diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml deleted file mode 100644 index 0500136fd..000000000 --- a/.github/workflows/cla.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Verify Contributor License Agreement - -on: [pull_request] - -jobs: - cla-validate: - - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Check CLA signing status for ${{ github.event.pull_request.user.login }} - run: | - cat > unsigned-cla.txt < -# Author: Juerg Haefliger -# -# This file is part of cloud-init. See LICENSE file for license information. - -"""Phone Home: Post data to url""" - -from textwrap import dedent - -from cloudinit import templater, url_helper, util -from cloudinit.config.schema import MetaSchema, get_meta_doc -from cloudinit.distros import ALL_DISTROS -from cloudinit.settings import PER_INSTANCE - -frequency = PER_INSTANCE - -POST_LIST_ALL = [ - "pub_key_dsa", - "pub_key_rsa", - "pub_key_ecdsa", - "pub_key_ed25519", - "instance_id", - "hostname", - "fqdn", -] - -MODULE_DESCRIPTION = """\ -This module can be used to post data to a remote host after boot is complete. -If the post url contains the string ``$INSTANCE_ID`` it will be replaced with -the id of the current instance. Either all data can be posted or a list of -keys to post. Available keys are: - - - ``pub_key_dsa`` - - ``pub_key_rsa`` - - ``pub_key_ecdsa`` - - ``pub_key_ed25519`` - - ``instance_id`` - - ``hostname`` - - ``fdqn`` - -Data is sent as ``x-www-form-urlencoded`` arguments. - -**Example HTTP POST**: - -.. code-block:: http - - POST / HTTP/1.1 - Content-Length: 1337 - User-Agent: Cloud-Init/21.4 - Accept-Encoding: gzip, deflate - Accept: */* - Content-Type: application/x-www-form-urlencoded - - pub_key_dsa=dsa_contents&pub_key_rsa=rsa_contents&pub_key_ecdsa=ecdsa_contents&pub_key_ed25519=ed25519_contents&instance_id=i-87018aed&hostname=myhost&fqdn=myhost.internal -""" - -meta: MetaSchema = { - "id": "cc_phone_home", - "name": "Phone Home", - "title": "Post data to url", - "description": MODULE_DESCRIPTION, - "distros": [ALL_DISTROS], - "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - phone_home: - url: http://example.com/$INSTANCE_ID/ - post: all - """ - ), - dedent( - """\ - phone_home: - url: http://example.com/$INSTANCE_ID/ - post: - - pub_key_dsa - - pub_key_rsa - - pub_key_ecdsa - - pub_key_ed25519 - - instance_id - - hostname - - fqdn - tries: 5 - """ - ), - ], -} - -__doc__ = get_meta_doc(meta) - -# phone_home: -# url: http://my.foo.bar/$INSTANCE/ -# post: all -# tries: 10 -# -# phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ -# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id, hostname, -# fqdn ] -# - - -def handle(name, cfg, cloud, log, args): - if len(args) != 0: - ph_cfg = util.read_conf(args[0]) - else: - if "phone_home" not in cfg: - log.debug( - "Skipping module named %s, " - "no 'phone_home' configuration found", - name, - ) - return - ph_cfg = cfg["phone_home"] - - if "url" not in ph_cfg: - log.warning( - "Skipping module named %s, " - "no 'url' found in 'phone_home' configuration", - name, - ) - return - - url = ph_cfg["url"] - post_list = ph_cfg.get("post", "all") - tries = ph_cfg.get("tries") - try: - tries = int(tries) # type: ignore - except ValueError: - tries = 10 - util.logexc( - log, - "Configuration entry 'tries' is not an integer, using %s instead", - tries, - ) - - if post_list == "all": - post_list = POST_LIST_ALL - - all_keys = {} - all_keys["instance_id"] = cloud.get_instance_id() - all_keys["hostname"] = cloud.get_hostname() - all_keys["fqdn"] = cloud.get_hostname(fqdn=True) - - pubkeys = { - "pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub", - "pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub", - "pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub", - "pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub", - } - - for (n, path) in pubkeys.items(): - try: - all_keys[n] = util.load_file(path) - except Exception: - util.logexc( - log, "%s: failed to open, can not phone home that data!", path - ) - - submit_keys = {} - for k in post_list: - if k in all_keys: - submit_keys[k] = all_keys[k] - else: - submit_keys[k] = None - log.warning( - "Requested key %s from 'post'" - " configuration list not available", - k, - ) - - # Get them read to be posted - real_submit_keys = {} - for (k, v) in submit_keys.items(): - if v is None: - real_submit_keys[k] = "N/A" - else: - real_submit_keys[k] = str(v) - - # Incase the url is parameterized - url_params = { - "INSTANCE_ID": all_keys["instance_id"], - } - url = templater.render_string(url, url_params) - try: - url_helper.read_file_or_url( - url, - data=real_submit_keys, - retries=tries, - sec_between=3, - ssl_details=util.fetch_ssl_details(cloud.paths), - ) - except Exception: - util.logexc( - log, "Failed to post phone home data to %s in %s tries", url, tries - ) - - -# vi: ts=4 expandtab diff --git a/.pc/cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500/tests/unittests/config/test_cc_phone_home.py b/.pc/cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500/tests/unittests/config/test_cc_phone_home.py deleted file mode 100644 index 7264dda1e..000000000 --- a/.pc/cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500/tests/unittests/config/test_cc_phone_home.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest - -from cloudinit.config.schema import ( - SchemaValidationError, - get_schema, - validate_cloudconfig_schema, -) -from tests.unittests.helpers import skipUnlessJsonSchema - - -class TestPhoneHomeSchema: - @pytest.mark.parametrize( - "config", - [ - # phone_home definition with url - {"phone_home": {"post": ["pub_key_dsa"]}}, - # post using string other than "all" - {"phone_home": {"url": "test_url", "post": "pub_key_dsa"}}, - # post using list with misspelled entry - {"phone_home": {"url": "test_url", "post": ["pub_kye_dsa"]}}, - ], - ) - @skipUnlessJsonSchema() - def test_schema_validation(self, config): - with pytest.raises(SchemaValidationError): - validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/cloudinit/cmd/main.py b/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/cloudinit/cmd/main.py deleted file mode 100755 index fcdaf7251..000000000 --- a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/cloudinit/cmd/main.py +++ /dev/null @@ -1,1075 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# Copyright (C) 2017 Amazon.com, Inc. or its affiliates -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# Author: Andrew Jorgensen -# -# This file is part of cloud-init. See LICENSE file for license information. - -# Skip isort on this file because of the patch that comes between imports -# isort: skip_file - -import argparse -import json -import os -import sys -import time -import traceback - -from cloudinit import patcher -from cloudinit.config.modules import Modules - -patcher.patch_logging() - -from cloudinit.config.schema import validate_cloudconfig_schema -from cloudinit import log as logging -from cloudinit import netinfo -from cloudinit import signal_handler -from cloudinit import sources -from cloudinit import stages -from cloudinit import url_helper -from cloudinit import util -from cloudinit import version -from cloudinit import warnings - -from cloudinit import reporting -from cloudinit.reporting import events - -from cloudinit.settings import PER_INSTANCE, PER_ALWAYS, PER_ONCE, CLOUD_CONFIG - -from cloudinit import atomic_helper - -from cloudinit.config import cc_set_hostname -from cloudinit import dhclient_hook - - -# Welcome message template -WELCOME_MSG_TPL = ( - "Cloud-init v. {version} running '{action}' at " - "{timestamp}. Up {uptime} seconds." -) - -# Module section template -MOD_SECTION_TPL = "cloud_%s_modules" - -# Frequency shortname to full name -# (so users don't have to remember the full name...) -FREQ_SHORT_NAMES = { - "instance": PER_INSTANCE, - "always": PER_ALWAYS, - "once": PER_ONCE, -} - -LOG = logging.getLogger() - - -# Used for when a logger may not be active -# and we still want to print exceptions... -def print_exc(msg=""): - if msg: - sys.stderr.write("%s\n" % (msg)) - sys.stderr.write("-" * 60) - sys.stderr.write("\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.write("-" * 60) - sys.stderr.write("\n") - - -def welcome(action, msg=None): - if not msg: - msg = welcome_format(action) - util.multi_log("%s\n" % (msg), console=False, stderr=True, log=LOG) - return msg - - -def welcome_format(action): - return WELCOME_MSG_TPL.format( - version=version.version_string(), - uptime=util.uptime(), - timestamp=util.time_rfc2822(), - action=action, - ) - - -def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... - fn_cfgs = [] - if args.files: - for fh in args.files: - # The realpath is more useful in logging - # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) - return fn_cfgs - - -def run_module_section(mods: Modules, action_name, section): - full_section_name = MOD_SECTION_TPL % (section) - (which_ran, failures) = mods.run_section(full_section_name) - total_attempted = len(which_ran) + len(failures) - if total_attempted == 0: - msg = "No '%s' modules to run under section '%s'" % ( - action_name, - full_section_name, - ) - sys.stderr.write("%s\n" % (msg)) - LOG.debug(msg) - return [] - else: - LOG.debug( - "Ran %s modules with %s failures", len(which_ran), len(failures) - ) - return failures - - -def apply_reporting_cfg(cfg): - if cfg.get("reporting"): - reporting.update_configuration(cfg.get("reporting")) - - -def parse_cmdline_url(cmdline, names=("cloud-config-url", "url")): - data = util.keyval_str_to_dict(cmdline) - for key in names: - if key in data: - return key, data[key] - raise KeyError("No keys (%s) found in string '%s'" % (cmdline, names)) - - -def attempt_cmdline_url(path, network=True, cmdline=None): - """Write data from url referenced in command line to path. - - path: a file to write content to if downloaded. - network: should network access be assumed. - cmdline: the cmdline to parse for cloud-config-url. - - This is used in MAAS datasource, in "ephemeral" (read-only root) - environment where the instance netboots to iscsi ro root. - and the entity that controls the pxe config has to configure - the maas datasource. - - An attempt is made on network urls even in local datasource - for case of network set up in initramfs. - - Return value is a tuple of a logger function (logging.DEBUG) - and a message indicating what happened. - """ - - if cmdline is None: - cmdline = util.get_cmdline() - - try: - cmdline_name, url = parse_cmdline_url(cmdline) - except KeyError: - return (logging.DEBUG, "No kernel command line url found.") - - path_is_local = url.startswith("file://") or url.startswith("/") - - if path_is_local and os.path.exists(path): - if network: - m = ( - "file '%s' existed, possibly from local stage download" - " of command line url '%s'. Not re-writing." % (path, url) - ) - level = logging.INFO - if path_is_local: - level = logging.DEBUG - else: - m = ( - "file '%s' existed, possibly from previous boot download" - " of command line url '%s'. Not re-writing." % (path, url) - ) - level = logging.WARN - - return (level, m) - - kwargs = {"url": url, "timeout": 10, "retries": 2} - if network or path_is_local: - level = logging.WARN - kwargs["sec_between"] = 1 - else: - level = logging.DEBUG - kwargs["sec_between"] = 0.1 - - data = None - header = b"#cloud-config" - try: - resp = url_helper.read_file_or_url(**kwargs) - if resp.ok(): - data = resp.contents - if not resp.contents.startswith(header): - if cmdline_name == "cloud-config-url": - level = logging.WARN - else: - level = logging.INFO - return ( - level, - "contents of '%s' did not start with %s" % (url, header), - ) - else: - return ( - level, - "url '%s' returned code %s. Ignoring." % (url, resp.code), - ) - - except url_helper.UrlError as e: - return (level, "retrieving url '%s' failed: %s" % (url, e)) - - util.write_file(path, data, mode=0o600) - return ( - logging.INFO, - "wrote cloud-config data from %s='%s' to %s" - % (cmdline_name, url, path), - ) - - -def purge_cache_on_python_version_change(init): - """Purge the cache if python version changed on us. - - There could be changes not represented in our cache (obj.pkl) after we - upgrade to a new version of python, so at that point clear the cache - """ - current_python_version = "%d.%d" % ( - sys.version_info.major, - sys.version_info.minor, - ) - python_version_path = os.path.join( - init.paths.get_cpath("data"), "python-version" - ) - if os.path.exists(python_version_path): - cached_python_version = open(python_version_path).read() - # The Python version has changed out from under us, anything that was - # pickled previously is likely useless due to API changes. - if cached_python_version != current_python_version: - LOG.debug("Python version change detected. Purging cache") - init.purge_cache(True) - util.write_file(python_version_path, current_python_version) - else: - if os.path.exists(init.paths.get_ipath_cur("obj_pkl")): - LOG.info( - "Writing python-version file. " - "Cache compatibility status is currently unknown." - ) - util.write_file(python_version_path, current_python_version) - - -def _should_bring_up_interfaces(init, args): - if util.get_cfg_option_bool(init.cfg, "disable_network_activation"): - return False - return not args.local - - -def main_init(name, args): - deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - if args.local: - deps = [sources.DEP_FILESYSTEM] - - early_logs = [ - attempt_cmdline_url( - path=os.path.join( - "%s.d" % CLOUD_CONFIG, "91_kernel_cmdline_url.cfg" - ), - network=not args.local, - ) - ] - - # Cloud-init 'init' stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Setup logging/output redirections with resultant config (if any) - # 3. Initialize the cloud-init filesystem - # 4. Check if we can stop early by looking for various files - # 5. Fetch the datasource - # 6. Connect to the current instance location + update the cache - # 7. Consume the userdata (handlers get activated here) - # 8. Construct the modules object - # 9. Adjust any subsequent logging/output redirections using the modules - # objects config as it may be different from init object - # 10. Run the modules for the 'init' stage - # 11. Done! - if not args.local: - w_msg = welcome_format(name) - else: - w_msg = welcome_format("%s-local" % (name)) - init = stages.Init(ds_deps=deps, reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - outfmt = None - errfmt = None - try: - early_logs.append((logging.DEBUG, "Closing stdin.")) - util.close_stdin() - (outfmt, errfmt) = util.fixup_output(init.cfg, name) - except Exception: - msg = "Failed to setup output redirection!" - util.logexc(LOG, msg) - print_exc(msg) - early_logs.append((logging.WARN, msg)) - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug( - "Logging being reset, this logger may no longer be active shortly" - ) - logging.resetLogging() - logging.setupLogging(init.cfg) - apply_reporting_cfg(init.cfg) - - # Any log usage prior to setupLogging above did not have local user log - # config applied. We send the welcome message now, as stderr/out have - # been redirected and log now configured. - welcome(name, msg=w_msg) - - # re-play early log messages before logging was setup - for lvl, msg in early_logs: - LOG.log(lvl, msg) - - # Stage 3 - try: - init.initialize() - except Exception: - util.logexc(LOG, "Failed to initialize, likely bad things to come!") - # Stage 4 - path_helper = init.paths - purge_cache_on_python_version_change(init) - mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK - - if mode == sources.DSMODE_NETWORK: - existing = "trust" - sys.stderr.write("%s\n" % (netinfo.debug_info())) - else: - existing = "check" - mcfg = util.get_cfg_option_bool(init.cfg, "manual_cache_clean", False) - if mcfg: - LOG.debug("manual cache clean set from config") - existing = "trust" - else: - mfile = path_helper.get_ipath_cur("manual_clean_marker") - if os.path.exists(mfile): - LOG.debug("manual cache clean found from marker: %s", mfile) - existing = "trust" - - init.purge_cache() - - # Stage 5 - bring_up_interfaces = _should_bring_up_interfaces(init, args) - try: - init.fetch(existing=existing) - # if in network mode, and the datasource is local - # then work was done at that stage. - if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: - LOG.debug( - "[%s] Exiting. datasource %s in local mode", - mode, - init.datasource, - ) - return (None, []) - except sources.DataSourceNotFoundException: - # In the case of 'cloud-init init' without '--local' it is a bit - # more likely that the user would consider it failure if nothing was - # found. - if mode == sources.DSMODE_LOCAL: - LOG.debug("No local datasource found") - else: - util.logexc( - LOG, "No instance datasource found! Likely bad things to come!" - ) - if not args.force: - init.apply_network_config(bring_up=bring_up_interfaces) - LOG.debug("[%s] Exiting without datasource", mode) - if mode == sources.DSMODE_LOCAL: - return (None, []) - else: - return (None, ["No instance datasource found."]) - else: - LOG.debug( - "[%s] barreling on in force mode without datasource", mode - ) - - _maybe_persist_instance_data(init) - # Stage 6 - iid = init.instancify() - LOG.debug( - "[%s] %s will now be targeting instance id: %s. new=%s", - mode, - name, - iid, - init.is_new_instance(), - ) - - if mode == sources.DSMODE_LOCAL: - # Before network comes up, set any configured hostname to allow - # dhcp clients to advertize this hostname to any DDNS services - # LP: #1746455. - _maybe_set_hostname(init, stage="local", retry_stage="network") - init.apply_network_config(bring_up=bring_up_interfaces) - - if mode == sources.DSMODE_LOCAL: - if init.datasource.dsmode != mode: - LOG.debug( - "[%s] Exiting. datasource %s not in local mode.", - mode, - init.datasource, - ) - return (init.datasource, []) - else: - LOG.debug( - "[%s] %s is in local mode, will apply init modules now.", - mode, - init.datasource, - ) - - # Give the datasource a chance to use network resources. - # This is used on Azure to communicate with the fabric over network. - init.setup_datasource() - # update fully realizes user-data (pulling in #include if necessary) - init.update() - _maybe_set_hostname(init, stage="init-net", retry_stage="modules:config") - # Stage 7 - try: - # Attempt to consume the data per instance. - # This may run user-data handlers and/or perform - # url downloads and such as needed. - (ran, _results) = init.cloudify().run( - "consume_data", - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - if not ran: - # Just consume anything that is set to run per-always - # if nothing ran in the per-instance code - # - # See: https://bugs.launchpad.net/bugs/819507 for a little - # reason behind this... - init.consume_data(PER_ALWAYS) - except Exception: - util.logexc(LOG, "Consuming user data failed!") - return (init.datasource, ["Consuming user data failed!"]) - - # Validate user-data adheres to schema definition - if os.path.exists(init.paths.get_ipath_cur("userdata_raw")): - validate_cloudconfig_schema(config=init.cfg, strict=False) - else: - LOG.debug("Skipping user-data validation. No user-data found.") - - apply_reporting_cfg(init.cfg) - - # Stage 8 - re-read and apply relevant cloud-config to include user-data - mods = Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 9 - try: - outfmt_orig = outfmt - errfmt_orig = errfmt - (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) - if outfmt_orig != outfmt or errfmt_orig != errfmt: - LOG.warning("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) - (outfmt, errfmt) = util.fixup_output(mods.cfg, name) - except Exception: - util.logexc(LOG, "Failed to re-adjust output redirection!") - logging.setupLogging(mods.cfg) - - # give the activated datasource a chance to adjust - init.activate_datasource() - - di_report_warn(datasource=init.datasource, cfg=init.cfg) - - # Stage 10 - return (init.datasource, run_module_section(mods, name, name)) - - -def di_report_warn(datasource, cfg): - if "di_report" not in cfg: - LOG.debug("no di_report found in config.") - return - - dicfg = cfg["di_report"] - if dicfg is None: - # ds-identify may write 'di_report:\n #comment\n' - # which reads as {'di_report': None} - LOG.debug("di_report was None.") - return - - if not isinstance(dicfg, dict): - LOG.warning("di_report config not a dictionary: %s", dicfg) - return - - dslist = dicfg.get("datasource_list") - if dslist is None: - LOG.warning("no 'datasource_list' found in di_report.") - return - elif not isinstance(dslist, list): - LOG.warning("di_report/datasource_list not a list: %s", dslist) - return - - # ds.__module__ is like cloudinit.sources.DataSourceName - # where Name is the thing that shows up in datasource_list. - modname = datasource.__module__.rpartition(".")[2] - if modname.startswith(sources.DS_PREFIX): - modname = modname[len(sources.DS_PREFIX) :] - else: - LOG.warning( - "Datasource '%s' came from unexpected module '%s'.", - datasource, - modname, - ) - - if modname in dslist: - LOG.debug( - "used datasource '%s' from '%s' was in di_report's list: %s", - datasource, - modname, - dslist, - ) - return - - warnings.show_warning( - "dsid_missing_source", cfg, source=modname, dslist=str(dslist) - ) - - -def main_modules(action_name, args): - name = args.mode - # Cloud-init 'modules' stages are broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Get the datasource from the init object, if it does - # not exist then that means the main_init stage never - # worked, and thus this stage can not run. - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the modules for the given stage name - # 6. Done! - w_msg = welcome_format("%s:%s" % (action_name, name)) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, theres nothing to do - msg = ( - "Can not apply stage %s, no datasource found! Likely bad " - "things to come!" % name - ) - util.logexc(LOG, msg) - print_exc(msg) - if not args.force: - return [(msg)] - _maybe_persist_instance_data(init) - # Stage 3 - mods = Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, name) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug( - "Logging being reset, this logger may no longer be active shortly" - ) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - return run_module_section(mods, name, name) - - -def main_single(name, args): - # Cloud-init single stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Attempt to fetch the datasource (warn if it doesn't work) - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the single module - # 6. Done! - mod_name = args.name - w_msg = welcome_format(name) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, - # that might be bad (or ok) depending on - # the module being ran (so continue on) - util.logexc( - LOG, "Failed to fetch your datasource, likely bad things to come!" - ) - print_exc( - "Failed to fetch your datasource, likely bad things to come!" - ) - if not args.force: - return 1 - _maybe_persist_instance_data(init) - # Stage 3 - mods = Modules(init, extract_fns(args), reporter=args.reporter) - mod_args = args.module_args - if mod_args: - LOG.debug("Using passed in arguments %s", mod_args) - mod_freq = args.frequency - if mod_freq: - LOG.debug("Using passed in frequency %s", mod_freq) - mod_freq = FREQ_SHORT_NAMES.get(mod_freq) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, None) - except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug( - "Logging being reset, this logger may no longer be active shortly" - ) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - (which_ran, failures) = mods.run_single(mod_name, mod_args, mod_freq) - if failures: - LOG.warning("Ran %s but it failed!", mod_name) - return 1 - elif not which_ran: - LOG.warning("Did not run %s, does it exist?", mod_name) - return 1 - else: - # Guess it worked - return 0 - - -def status_wrapper(name, args, data_d=None, link_d=None): - if data_d is None: - data_d = os.path.normpath("/var/lib/cloud/data") - if link_d is None: - link_d = os.path.normpath("/run/cloud-init") - - status_path = os.path.join(data_d, "status.json") - status_link = os.path.join(link_d, "status.json") - result_path = os.path.join(data_d, "result.json") - result_link = os.path.join(link_d, "result.json") - - util.ensure_dirs( - ( - data_d, - link_d, - ) - ) - - (_name, functor) = args.action - - if name == "init": - if args.local: - mode = "init-local" - else: - mode = "init" - elif name == "modules": - mode = "modules-%s" % args.mode - else: - raise ValueError("unknown name: %s" % name) - - modes = ( - "init", - "init-local", - "modules-init", - "modules-config", - "modules-final", - ) - if mode not in modes: - raise ValueError( - "Invalid cloud init mode specified '{0}'".format(mode) - ) - - status = None - if mode == "init-local": - for f in (status_link, result_link, status_path, result_path): - util.del_file(f) - else: - try: - status = json.loads(util.load_file(status_path)) - except Exception: - pass - - nullstatus = { - "errors": [], - "start": None, - "finished": None, - } - - if status is None: - status = {"v1": {}} - status["v1"]["datasource"] = None - - for m in modes: - if m not in status["v1"]: - status["v1"][m] = nullstatus.copy() - - v1 = status["v1"] - v1["stage"] = mode - v1[mode]["start"] = time.time() - - atomic_helper.write_json(status_path, status) - util.sym_link( - os.path.relpath(status_path, link_d), status_link, force=True - ) - - try: - ret = functor(name, args) - if mode in ("init", "init-local"): - (datasource, errors) = ret - if datasource is not None: - v1["datasource"] = str(datasource) - else: - errors = ret - - v1[mode]["errors"] = [str(e) for e in errors] - - except Exception as e: - util.logexc(LOG, "failed stage %s", mode) - print_exc("failed run of stage %s" % mode) - v1[mode]["errors"] = [str(e)] - - v1[mode]["finished"] = time.time() - v1["stage"] = None - - atomic_helper.write_json(status_path, status) - - if mode == "modules-final": - # write the 'finished' file - errors = [] - for m in modes: - if v1[m]["errors"]: - errors.extend(v1[m].get("errors", [])) - - atomic_helper.write_json( - result_path, - {"v1": {"datasource": v1["datasource"], "errors": errors}}, - ) - util.sym_link( - os.path.relpath(result_path, link_d), result_link, force=True - ) - - return len(v1[mode]["errors"]) - - -def _maybe_persist_instance_data(init): - """Write instance-data.json file if absent and datasource is restored.""" - if init.ds_restored: - instance_data_file = os.path.join( - init.paths.run_dir, sources.INSTANCE_JSON_FILE - ) - if not os.path.exists(instance_data_file): - init.datasource.persist_instance_data() - - -def _maybe_set_hostname(init, stage, retry_stage): - """Call set-hostname if metadata, vendordata or userdata provides it. - - @param stage: String representing current stage in which we are running. - @param retry_stage: String represented logs upon error setting hostname. - """ - cloud = init.cloudify() - (hostname, _fqdn) = util.get_hostname_fqdn( - init.cfg, cloud, metadata_only=True - ) - if hostname: # meta-data or user-data hostname content - try: - cc_set_hostname.handle("set-hostname", init.cfg, cloud, LOG, None) - except cc_set_hostname.SetHostnameError as e: - LOG.debug( - "Failed setting hostname in %s stage. Will" - " retry in %s stage. Error: %s.", - stage, - retry_stage, - str(e), - ) - - -def main_features(name, args): - sys.stdout.write("\n".join(sorted(version.FEATURES)) + "\n") - - -def main(sysv_args=None): - if not sysv_args: - sysv_args = sys.argv - parser = argparse.ArgumentParser(prog=sysv_args.pop(0)) - - # Top level args - parser.add_argument( - "--version", - "-v", - action="version", - version="%(prog)s " + (version.version_string()), - help="Show program's version number and exit.", - ) - parser.add_argument( - "--file", - "-f", - action="append", - dest="files", - help="Use additional yaml configuration files.", - type=argparse.FileType("rb"), - ) - parser.add_argument( - "--debug", - "-d", - action="store_true", - help="Show additional pre-action logging (default: %(default)s).", - default=False, - ) - parser.add_argument( - "--force", - action="store_true", - help=( - "Force running even if no datasource is" - " found (use at your own risk)." - ), - dest="force", - default=False, - ) - - parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") - subparsers.required = True - - # Each action and its sub-options (if any) - parser_init = subparsers.add_parser( - "init", help="Initialize cloud-init and perform initial modules." - ) - parser_init.add_argument( - "--local", - "-l", - action="store_true", - help="Start in local mode (default: %(default)s).", - default=False, - ) - # This is used so that we can know which action is selected + - # the functor to use to run this subcommand - parser_init.set_defaults(action=("init", main_init)) - - # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser( - "modules", help="Activate modules using a given configuration key." - ) - parser_mod.add_argument( - "--mode", - "-m", - action="store", - help="Module configuration name to use (default: %(default)s).", - default="config", - choices=("init", "config", "final"), - ) - parser_mod.set_defaults(action=("modules", main_modules)) - - # This subcommand allows you to run a single module - parser_single = subparsers.add_parser( - "single", help="Run a single module." - ) - parser_single.add_argument( - "--name", - "-n", - action="store", - help="module name to run", - required=True, - ) - parser_single.add_argument( - "--frequency", - action="store", - help="Set module frequency.", - required=False, - choices=list(FREQ_SHORT_NAMES.keys()), - ) - parser_single.add_argument( - "--report", - action="store_true", - help="Enable reporting.", - required=False, - ) - parser_single.add_argument( - "module_args", - nargs="*", - metavar="argument", - help="Any additional arguments to pass to this module.", - ) - parser_single.set_defaults(action=("single", main_single)) - - parser_query = subparsers.add_parser( - "query", - help="Query standardized instance metadata from the command line.", - ) - - parser_dhclient = subparsers.add_parser( - dhclient_hook.NAME, help=dhclient_hook.__doc__ - ) - dhclient_hook.get_parser(parser_dhclient) - - parser_features = subparsers.add_parser( - "features", help="List defined features." - ) - parser_features.set_defaults(action=("features", main_features)) - - parser_analyze = subparsers.add_parser( - "analyze", help="Devel tool: Analyze cloud-init logs and data." - ) - - parser_devel = subparsers.add_parser( - "devel", help="Run development tools." - ) - - parser_collect_logs = subparsers.add_parser( - "collect-logs", help="Collect and tar all cloud-init debug info." - ) - - parser_clean = subparsers.add_parser( - "clean", help="Remove logs and artifacts so cloud-init can re-run." - ) - - parser_status = subparsers.add_parser( - "status", help="Report cloud-init status or wait on completion." - ) - - parser_schema = subparsers.add_parser( - "schema", help="Validate cloud-config files using jsonschema." - ) - - if sysv_args: - # Only load subparsers if subcommand is specified to avoid load cost - subcommand = sysv_args[0] - if subcommand == "analyze": - from cloudinit.analyze.__main__ import get_parser as analyze_parser - - # Construct analyze subcommand parser - analyze_parser(parser_analyze) - elif subcommand == "devel": - from cloudinit.cmd.devel.parser import get_parser as devel_parser - - # Construct devel subcommand parser - devel_parser(parser_devel) - elif subcommand == "collect-logs": - from cloudinit.cmd.devel.logs import ( - get_parser as logs_parser, - handle_collect_logs_args, - ) - - logs_parser(parser_collect_logs) - parser_collect_logs.set_defaults( - action=("collect-logs", handle_collect_logs_args) - ) - elif subcommand == "clean": - from cloudinit.cmd.clean import ( - get_parser as clean_parser, - handle_clean_args, - ) - - clean_parser(parser_clean) - parser_clean.set_defaults(action=("clean", handle_clean_args)) - elif subcommand == "query": - from cloudinit.cmd.query import ( - get_parser as query_parser, - handle_args as handle_query_args, - ) - - query_parser(parser_query) - parser_query.set_defaults(action=("render", handle_query_args)) - elif subcommand == "schema": - from cloudinit.config.schema import ( - get_parser as schema_parser, - handle_schema_args, - ) - - schema_parser(parser_schema) - parser_schema.set_defaults(action=("schema", handle_schema_args)) - elif subcommand == "status": - from cloudinit.cmd.status import ( - get_parser as status_parser, - handle_status_args, - ) - - status_parser(parser_status) - parser_status.set_defaults(action=("status", handle_status_args)) - - args = parser.parse_args(args=sysv_args) - - # Subparsers.required = True and each subparser sets action=(name, functor) - (name, functor) = args.action - - # Setup basic logging to start (until reinitialized) - # iff in debug mode. - if args.debug: - logging.setupBasicLogging() - - # Setup signal handlers before running - signal_handler.attach_handlers() - - if name in ("modules", "init"): - functor = status_wrapper - - rname = None - report_on = True - if name == "init": - if args.local: - rname, rdesc = ("init-local", "searching for local datasources") - else: - rname, rdesc = ( - "init-network", - "searching for network datasources", - ) - elif name == "modules": - rname, rdesc = ( - "modules-%s" % args.mode, - "running modules for %s" % args.mode, - ) - elif name == "single": - rname, rdesc = ( - "single/%s" % args.name, - "running single module %s" % args.name, - ) - report_on = args.report - else: - rname = name - rdesc = "running 'cloud-init %s'" % name - report_on = False - - args.reporter = events.ReportEventStack( - rname, rdesc, reporting_enabled=report_on - ) - - with args.reporter: - retval = util.log_time( - logfunc=LOG.debug, - msg="cloud-init mode '%s'" % name, - get_uptime=True, - func=functor, - args=(name, args), - ) - reporting.flush_events() - return retval - - -if __name__ == "__main__": - if "TZ" not in os.environ: - os.environ["TZ"] = ":/etc/localtime" - return_value = main(sys.argv) - if return_value: - sys.exit(return_value) diff --git a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/cloudinit/config/schema.py b/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/cloudinit/config/schema.py deleted file mode 100644 index 7a6ecf08c..000000000 --- a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/cloudinit/config/schema.py +++ /dev/null @@ -1,788 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. -"""schema.py: Set of module functions for processing cloud-config schema.""" - -import argparse -import json -import logging -import os -import re -import sys -import typing -from collections import defaultdict -from copy import deepcopy -from functools import partial - -import yaml - -from cloudinit import importer, safeyaml -from cloudinit.cmd.devel import read_cfg_paths -from cloudinit.util import error, find_modules, load_file - -error = partial(error, sys_exit=True) -LOG = logging.getLogger(__name__) - -VERSIONED_USERDATA_SCHEMA_FILE = "versions.schema.cloud-config.json" -# Bump this file when introducing incompatible schema changes. -# Also add new version definition to versions.schema.json. -USERDATA_SCHEMA_FILE = "schema-cloud-config-v1.json" -_YAML_MAP = {True: "true", False: "false", None: "null"} -CLOUD_CONFIG_HEADER = b"#cloud-config" -SCHEMA_DOC_TMPL = """ -{name} -{title_underbar} -**Summary:** {title} - -{description} - -**Internal name:** ``{id}`` - -**Module frequency:** {frequency} - -**Supported distros:** {distros} - -{property_header} -{property_doc} - -{examples} -""" -SCHEMA_PROPERTY_HEADER = "**Config schema**:" -SCHEMA_PROPERTY_TMPL = "{prefix}**{prop_name}:** ({prop_type}){description}" -SCHEMA_LIST_ITEM_TMPL = ( - "{prefix}Each object in **{prop_name}** list supports the following keys:" -) -SCHEMA_EXAMPLES_HEADER = "**Examples**::\n\n" -SCHEMA_EXAMPLES_SPACER_TEMPLATE = "\n # --- Example{0} ---" - - -# annotations add value for development, but don't break old versions -# pyver: 3.6 -> 3.8 -# pylint: disable=E1101 -if sys.version_info >= (3, 8): - - class MetaSchema(typing.TypedDict): - name: str - id: str - title: str - description: str - distros: typing.List[str] - examples: typing.List[str] - frequency: str - -else: - MetaSchema = dict -# pylint: enable=E1101 - - -class SchemaValidationError(ValueError): - """Raised when validating a cloud-config file against a schema.""" - - def __init__(self, schema_errors=()): - """Init the exception an n-tuple of schema errors. - - @param schema_errors: An n-tuple of the format: - ((flat.config.key, msg),) - """ - self.schema_errors = schema_errors - error_messages = [ - "{0}: {1}".format(config_key, message) - for config_key, message in schema_errors - ] - message = "Cloud config schema errors: {0}".format( - ", ".join(error_messages) - ) - super(SchemaValidationError, self).__init__(message) - - -def is_schema_byte_string(checker, instance): - """TYPE_CHECKER override allowing bytes for string type - - For jsonschema v. 3.0.0+ - """ - try: - from jsonschema import Draft4Validator - except ImportError: - return False - return Draft4Validator.TYPE_CHECKER.is_type( - instance, "string" - ) or isinstance(instance, (bytes,)) - - -def get_jsonschema_validator(): - """Get metaschema validator and format checker - - Older versions of jsonschema require some compatibility changes. - - @returns: Tuple: (jsonschema.Validator, FormatChecker) - @raises: ImportError when jsonschema is not present - """ - from jsonschema import Draft4Validator, FormatChecker - from jsonschema.validators import create - - # Allow for bytes to be presented as an acceptable valid value for string - # type jsonschema attributes in cloud-init's schema. - # This allows #cloud-config to provide valid yaml "content: !!binary | ..." - - strict_metaschema = deepcopy(Draft4Validator.META_SCHEMA) - strict_metaschema["additionalProperties"] = False - - # This additional label allows us to specify a different name - # than the property key when generating docs. - # This is especially useful when using a "patternProperties" regex, - # otherwise the property label in the generated docs will be a - # regular expression. - # http://json-schema.org/understanding-json-schema/reference/object.html#pattern-properties - strict_metaschema["properties"]["label"] = {"type": "string"} - - if hasattr(Draft4Validator, "TYPE_CHECKER"): # jsonschema 3.0+ - type_checker = Draft4Validator.TYPE_CHECKER.redefine( - "string", is_schema_byte_string - ) - cloudinitValidator = create( - meta_schema=strict_metaschema, - validators=Draft4Validator.VALIDATORS, - version="draft4", - type_checker=type_checker, - ) - else: # jsonschema 2.6 workaround - types = Draft4Validator.DEFAULT_TYPES # pylint: disable=E1101 - # Allow bytes as well as string (and disable a spurious unsupported - # assignment-operation pylint warning which appears because this - # code path isn't written against the latest jsonschema). - types["string"] = (str, bytes) # pylint: disable=E1137 - cloudinitValidator = create( # pylint: disable=E1123 - meta_schema=strict_metaschema, - validators=Draft4Validator.VALIDATORS, - version="draft4", - default_types=types, - ) - return (cloudinitValidator, FormatChecker) - - -def validate_cloudconfig_metaschema(validator, schema: dict, throw=True): - """Validate provided schema meets the metaschema definition. Return strict - Validator and FormatChecker for use in validation - @param validator: Draft4Validator instance used to validate the schema - @param schema: schema to validate - @param throw: Sometimes the validator and checker are required, even if - the schema is invalid. Toggle for whether to raise - SchemaValidationError or log warnings. - - @raises: ImportError when jsonschema is not present - @raises: SchemaValidationError when the schema is invalid - """ - - from jsonschema.exceptions import SchemaError - - try: - validator.check_schema(schema) - except SchemaError as err: - # Raise SchemaValidationError to avoid jsonschema imports at call - # sites - if throw: - raise SchemaValidationError( - schema_errors=( - (".".join([str(p) for p in err.path]), err.message), - ) - ) from err - LOG.warning( - "Meta-schema validation failed, attempting to validate config " - "anyway: %s", - err, - ) - - -def validate_cloudconfig_schema( - config: dict, - schema: dict = None, - strict: bool = False, - strict_metaschema: bool = False, -): - """Validate provided config meets the schema definition. - - @param config: Dict of cloud configuration settings validated against - schema. Ignored if strict_metaschema=True - @param schema: jsonschema dict describing the supported schema definition - for the cloud config module (config.cc_*). If None, validate against - global schema. - @param strict: Boolean, when True raise SchemaValidationErrors instead of - logging warnings. - @param strict_metaschema: Boolean, when True validates schema using strict - metaschema definition at runtime (currently unused) - - @raises: SchemaValidationError when provided config does not validate - against the provided schema. - @raises: RuntimeError when provided config sourced from YAML is not a dict. - """ - if schema is None: - schema = get_schema() - try: - (cloudinitValidator, FormatChecker) = get_jsonschema_validator() - if strict_metaschema: - validate_cloudconfig_metaschema( - cloudinitValidator, schema, throw=False - ) - except ImportError: - LOG.debug("Ignoring schema validation. jsonschema is not present") - return - - validator = cloudinitValidator(schema, format_checker=FormatChecker()) - errors = () - for error in sorted(validator.iter_errors(config), key=lambda e: e.path): - path = ".".join([str(p) for p in error.path]) - errors += ((path, error.message),) - if errors: - if strict: - raise SchemaValidationError(errors) - else: - messages = ["{0}: {1}".format(k, msg) for k, msg in errors] - LOG.warning( - "Invalid cloud-config provided:\n%s", "\n".join(messages) - ) - - -def annotated_cloudconfig_file( - cloudconfig, original_content, schema_errors, schemamarks -): - """Return contents of the cloud-config file annotated with schema errors. - - @param cloudconfig: YAML-loaded dict from the original_content or empty - dict if unparseable. - @param original_content: The contents of a cloud-config file - @param schema_errors: List of tuples from a JSONSchemaValidationError. The - tuples consist of (schemapath, error_message). - """ - if not schema_errors: - return original_content - errors_by_line = defaultdict(list) - error_footer = [] - error_header = "# Errors: -------------\n{0}\n\n" - annotated_content = [] - lines = original_content.decode().split("\n") - if not isinstance(cloudconfig, dict): - # Return a meaningful message on empty cloud-config - return "\n".join( - lines - + [error_header.format("# E1: Cloud-config is not a YAML dict.")] - ) - for path, msg in schema_errors: - match = re.match(r"format-l(?P\d+)\.c(?P\d+).*", path) - if match: - line, col = match.groups() - errors_by_line[int(line)].append(msg) - else: - col = None - errors_by_line[schemamarks[path]].append(msg) - if col is not None: - msg = "Line {line} column {col}: {msg}".format( - line=line, col=col, msg=msg - ) - error_index = 1 - for line_number, line in enumerate(lines, 1): - errors = errors_by_line[line_number] - if errors: - error_label = [] - for error in errors: - error_label.append("E{0}".format(error_index)) - error_footer.append("# E{0}: {1}".format(error_index, error)) - error_index += 1 - annotated_content.append(line + "\t\t# " + ",".join(error_label)) - - else: - annotated_content.append(line) - annotated_content.append(error_header.format("\n".join(error_footer))) - return "\n".join(annotated_content) - - -def validate_cloudconfig_file(config_path, schema, annotate=False): - """Validate cloudconfig file adheres to a specific jsonschema. - - @param config_path: Path to the yaml cloud-config file to parse, or None - to default to system userdata from Paths object. - @param schema: Dict describing a valid jsonschema to validate against. - @param annotate: Boolean set True to print original config file with error - annotations on the offending lines. - - @raises SchemaValidationError containing any of schema_errors encountered. - @raises RuntimeError when config_path does not exist. - """ - if config_path is None: - # Use system's raw userdata path - if os.getuid() != 0: - raise RuntimeError( - "Unable to read system userdata as non-root user." - " Try using sudo" - ) - paths = read_cfg_paths() - user_data_file = paths.get_ipath_cur("userdata_raw") - content = load_file(user_data_file, decode=False) - else: - if not os.path.exists(config_path): - raise RuntimeError( - "Configfile {0} does not exist".format(config_path) - ) - content = load_file(config_path, decode=False) - if not content.startswith(CLOUD_CONFIG_HEADER): - errors = ( - ( - "format-l1.c1", - 'File {0} needs to begin with "{1}"'.format( - config_path, CLOUD_CONFIG_HEADER.decode() - ), - ), - ) - error = SchemaValidationError(errors) - if annotate: - print( - annotated_cloudconfig_file( - {}, content, error.schema_errors, {} - ) - ) - raise error - try: - if annotate: - cloudconfig, marks = safeyaml.load_with_marks(content) - else: - cloudconfig = safeyaml.load(content) - marks = {} - except (yaml.YAMLError) as e: - line = column = 1 - mark = None - if hasattr(e, "context_mark") and getattr(e, "context_mark"): - mark = getattr(e, "context_mark") - elif hasattr(e, "problem_mark") and getattr(e, "problem_mark"): - mark = getattr(e, "problem_mark") - if mark: - line = mark.line + 1 - column = mark.column + 1 - errors = ( - ( - "format-l{line}.c{col}".format(line=line, col=column), - "File {0} is not valid yaml. {1}".format(config_path, str(e)), - ), - ) - error = SchemaValidationError(errors) - if annotate: - print( - annotated_cloudconfig_file( - {}, content, error.schema_errors, {} - ) - ) - raise error from e - if not isinstance(cloudconfig, dict): - # Return a meaningful message on empty cloud-config - if not annotate: - raise RuntimeError("Cloud-config is not a YAML dict.") - try: - validate_cloudconfig_schema(cloudconfig, schema, strict=True) - except SchemaValidationError as e: - if annotate: - print( - annotated_cloudconfig_file( - cloudconfig, content, e.schema_errors, marks - ) - ) - raise - - -def _sort_property_order(value): - """Provide a sorting weight for documentation of property types. - - Weight values ensure 'array' sorted after 'object' which is sorted - after anything else which remains unsorted. - """ - if value == "array": - return 2 - elif value == "object": - return 1 - return 0 - - -def _get_property_type(property_dict: dict, defs: dict) -> str: - """Return a string representing a property type from a given - jsonschema. - """ - _flatten_schema_refs(property_dict, defs) - property_types = property_dict.get("type", []) - if not isinstance(property_types, list): - property_types = [property_types] - if property_dict.get("enum"): - property_types = [ - f"``{_YAML_MAP.get(k, k)}``" for k in property_dict["enum"] - ] - elif property_dict.get("oneOf"): - property_types.extend( - [ - subschema["type"] - for subschema in property_dict.get("oneOf") - if subschema.get("type") - ] - ) - if len(property_types) == 1: - property_type = property_types[0] - else: - property_types.sort(key=_sort_property_order) - property_type = "/".join(property_types) - items = property_dict.get("items", {}) - sub_property_types = items.get("type", []) - if not isinstance(sub_property_types, list): - sub_property_types = [sub_property_types] - # Collect each item type - for sub_item in items.get("oneOf", {}): - sub_property_types.append(_get_property_type(sub_item, defs)) - if sub_property_types: - if len(sub_property_types) == 1: - return f"{property_type} of {sub_property_types[0]}" - sub_property_types.sort(key=_sort_property_order) - sub_property_doc = f"({'/'.join(sub_property_types)})" - return f"{property_type} of {sub_property_doc}" - return property_type or "UNDEFINED" - - -def _parse_description(description, prefix) -> str: - """Parse description from the meta in a format that we can better - display in our docs. This parser does three things: - - - Guarantee that a paragraph will be in a single line - - Guarantee that each new paragraph will be aligned with - the first paragraph - - Proper align lists of items - - @param description: The original description in the meta. - @param prefix: The number of spaces used to align the current description - """ - list_paragraph = prefix * 3 - description = re.sub(r"(\S)\n(\S)", r"\1 \2", description) - description = re.sub(r"\n\n", r"\n\n{}".format(prefix), description) - description = re.sub( - r"\n( +)-", r"\n{}-".format(list_paragraph), description - ) - - return description - - -def _flatten_schema_refs(src_cfg: dict, defs: dict): - """Flatten schema: replace $refs in src_cfg with definitions from $defs.""" - if "$ref" in src_cfg: - reference = src_cfg.pop("$ref").replace("#/$defs/", "") - # Update the defined references in subschema for doc rendering - src_cfg.update(defs[reference]) - if "items" in src_cfg: - if "$ref" in src_cfg["items"]: - reference = src_cfg["items"].pop("$ref").replace("#/$defs/", "") - # Update the references in subschema for doc rendering - src_cfg["items"].update(defs[reference]) - if "oneOf" in src_cfg["items"]: - for alt_schema in src_cfg["items"]["oneOf"]: - if "$ref" in alt_schema: - reference = alt_schema.pop("$ref").replace("#/$defs/", "") - alt_schema.update(defs[reference]) - for alt_schema in src_cfg.get("oneOf", []): - if "$ref" in alt_schema: - reference = alt_schema.pop("$ref").replace("#/$defs/", "") - alt_schema.update(defs[reference]) - - -def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: - """Return restructured text describing the supported schema properties.""" - new_prefix = prefix + " " - properties = [] - if schema.get("hidden") is True: - return "" # no docs for this schema - property_keys = [ - key - for key in ("properties", "patternProperties") - if "hidden" not in schema or key not in schema["hidden"] - ] - property_schemas = [schema.get(key, {}) for key in property_keys] - - for prop_schema in property_schemas: - for prop_key, prop_config in prop_schema.items(): - _flatten_schema_refs(prop_config, defs) - if prop_config.get("hidden") is True: - continue # document nothing for this property - # Define prop_name and description for SCHEMA_PROPERTY_TMPL - description = prop_config.get("description", "") - if description: - description = " " + description - - # Define prop_name and description for SCHEMA_PROPERTY_TMPL - label = prop_config.get("label", prop_key) - properties.append( - SCHEMA_PROPERTY_TMPL.format( - prefix=prefix, - prop_name=label, - description=_parse_description(description, prefix), - prop_type=_get_property_type(prop_config, defs), - ) - ) - items = prop_config.get("items") - if items: - _flatten_schema_refs(items, defs) - if items.get("properties") or items.get("patternProperties"): - properties.append( - SCHEMA_LIST_ITEM_TMPL.format( - prefix=new_prefix, prop_name=label - ) - ) - new_prefix += " " - properties.append( - _get_property_doc(items, defs=defs, prefix=new_prefix) - ) - for alt_schema in items.get("oneOf", []): - if alt_schema.get("properties") or alt_schema.get( - "patternProperties" - ): - properties.append( - SCHEMA_LIST_ITEM_TMPL.format( - prefix=new_prefix, prop_name=label - ) - ) - new_prefix += " " - properties.append( - _get_property_doc( - alt_schema, defs=defs, prefix=new_prefix - ) - ) - if ( - "properties" in prop_config - or "patternProperties" in prop_config - ): - properties.append( - _get_property_doc( - prop_config, defs=defs, prefix=new_prefix - ) - ) - return "\n\n".join(properties) - - -def _get_examples(meta: MetaSchema) -> str: - """Return restructured text describing the meta examples if present.""" - examples = meta.get("examples") - if not examples: - return "" - rst_content = SCHEMA_EXAMPLES_HEADER - for count, example in enumerate(examples): - # Python2.6 is missing textwrapper.indent - lines = example.split("\n") - indented_lines = [" {0}".format(line) for line in lines] - if rst_content != SCHEMA_EXAMPLES_HEADER: - indented_lines.insert( - 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1) - ) - rst_content += "\n".join(indented_lines) - return rst_content - - -def get_meta_doc(meta: MetaSchema, schema: dict = None) -> str: - """Return reStructured text rendering the provided metadata. - - @param meta: Dict of metadata to render. - @param schema: Optional module schema, if absent, read global schema. - @raise KeyError: If metadata lacks an expected key. - """ - - if schema is None: - schema = get_schema() - if not meta or not schema: - raise ValueError("Expected non-empty meta and schema") - keys = set(meta.keys()) - expected = set( - { - "id", - "title", - "examples", - "frequency", - "distros", - "description", - "name", - } - ) - error_message = "" - if expected - keys: - error_message = "Missing expected keys in module meta: {}".format( - expected - keys - ) - elif keys - expected: - error_message = ( - "Additional unexpected keys found in module meta: {}".format( - keys - expected - ) - ) - if error_message: - raise KeyError(error_message) - - # cast away type annotation - meta_copy = dict(deepcopy(meta)) - meta_copy["property_header"] = "" - defs = schema.get("$defs", {}) - if defs.get(meta["id"]): - schema = defs.get(meta["id"]) - try: - meta_copy["property_doc"] = _get_property_doc(schema, defs=defs) - except AttributeError: - LOG.warning("Unable to render property_doc due to invalid schema") - meta_copy["property_doc"] = "" - if meta_copy["property_doc"]: - meta_copy["property_header"] = SCHEMA_PROPERTY_HEADER - meta_copy["examples"] = _get_examples(meta) - meta_copy["distros"] = ", ".join(meta["distros"]) - # Need an underbar of the same length as the name - meta_copy["title_underbar"] = re.sub(r".", "-", meta["name"]) - template = SCHEMA_DOC_TMPL.format(**meta_copy) - return template - - -def get_modules() -> dict: - configs_dir = os.path.dirname(os.path.abspath(__file__)) - return find_modules(configs_dir) - - -def load_doc(requested_modules: list) -> str: - """Load module docstrings - - Docstrings are generated on module load. Reduce, reuse, recycle. - """ - docs = "" - all_modules = list(get_modules().values()) + ["all"] - invalid_docs = set(requested_modules).difference(set(all_modules)) - if invalid_docs: - error( - "Invalid --docs value {}. Must be one of: {}".format( - list(invalid_docs), - ", ".join(all_modules), - ) - ) - for mod_name in all_modules: - if "all" in requested_modules or mod_name in requested_modules: - (mod_locs, _) = importer.find_module( - mod_name, ["cloudinit.config"], ["meta"] - ) - if mod_locs: - mod = importer.import_module(mod_locs[0]) - docs += mod.__doc__ or "" - return docs - - -def get_schema_dir() -> str: - return os.path.join(os.path.dirname(os.path.abspath(__file__)), "schemas") - - -def get_schema() -> dict: - """Return jsonschema coalesced from all cc_* cloud-config modules.""" - # Note versions.schema.json is publicly consumed by schemastore.org. - # If we change the location of versions.schema.json in github, we need - # to provide an updated PR to - # https://github.com/SchemaStore/schemastore. - - # When bumping schema version due to incompatible changes: - # 1. Add a new schema-cloud-config-v#.json - # 2. change the USERDATA_SCHEMA_FILE to cloud-init-schema-v#.json - # 3. Add the new version definition to versions.schema.cloud-config.json - schema_file = os.path.join(get_schema_dir(), USERDATA_SCHEMA_FILE) - full_schema = None - try: - full_schema = json.loads(load_file(schema_file)) - except Exception as e: - LOG.warning("Cannot parse JSON schema file %s. %s", schema_file, e) - if not full_schema: - LOG.warning( - "No base JSON schema files found at %s." - " Setting default empty schema", - schema_file, - ) - full_schema = { - "$defs": {}, - "$schema": "http://json-schema.org/draft-04/schema#", - "allOf": [], - } - return full_schema - - -def get_meta() -> dict: - """Return metadata coalesced from all cc_* cloud-config module.""" - full_meta = dict() - for (_, mod_name) in get_modules().items(): - mod_locs, _ = importer.find_module( - mod_name, ["cloudinit.config"], ["meta"] - ) - if mod_locs: - mod = importer.import_module(mod_locs[0]) - full_meta[mod.meta["id"]] = mod.meta - return full_meta - - -def get_parser(parser=None): - """Return a parser for supported cmdline arguments.""" - if not parser: - parser = argparse.ArgumentParser( - prog="cloudconfig-schema", - description="Validate cloud-config files or document schema", - ) - parser.add_argument( - "-c", - "--config-file", - help="Path of the cloud-config yaml file to validate", - ) - parser.add_argument( - "--system", - action="store_true", - default=False, - help="Validate the system cloud-config userdata", - ) - parser.add_argument( - "-d", - "--docs", - nargs="+", - help=( - "Print schema module docs. Choices: all or" - " space-delimited cc_names." - ), - ) - parser.add_argument( - "--annotate", - action="store_true", - default=False, - help="Annotate existing cloud-config file with errors", - ) - return parser - - -def handle_schema_args(name, args): - """Handle provided schema args and perform the appropriate actions.""" - exclusive_args = [args.config_file, args.docs, args.system] - if len([arg for arg in exclusive_args if arg]) != 1: - error("Expected one of --config-file, --system or --docs arguments") - if args.annotate and args.docs: - error("Invalid flag combination. Cannot use --annotate with --docs") - full_schema = get_schema() - if args.config_file or args.system: - try: - validate_cloudconfig_file( - args.config_file, full_schema, args.annotate - ) - except SchemaValidationError as e: - if not args.annotate: - error(str(e)) - except RuntimeError as e: - error(str(e)) - else: - if args.config_file is None: - cfg_name = "system userdata" - else: - cfg_name = args.config_file - print("Valid cloud-config:", cfg_name) - elif args.docs: - print(load_doc(args.docs)) - - -def main(): - """Tool to validate schema of a cloud-config file or print schema docs.""" - parser = get_parser() - handle_schema_args("cloudconfig-schema", parser.parse_args()) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) - -# vi: ts=4 expandtab diff --git a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/tests/integration_tests/modules/test_cli.py b/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/tests/integration_tests/modules/test_cli.py deleted file mode 100644 index e878176f4..000000000 --- a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/tests/integration_tests/modules/test_cli.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Integration tests for CLI functionality - -These would be for behavior manually invoked by user from the command line -""" - -import pytest - -from tests.integration_tests.instances import IntegrationInstance - -VALID_USER_DATA = """\ -#cloud-config -runcmd: - - echo 'hi' > /var/tmp/test -""" - -INVALID_USER_DATA_HEADER = """\ -runcmd: - - echo 'hi' > /var/tmp/test -""" - -INVALID_USER_DATA_SCHEMA = """\ -#cloud-config -updates: - notnetwork: -1 -apt_pipelining: bogus -""" - - -@pytest.mark.user_data(VALID_USER_DATA) -def test_valid_userdata(client: IntegrationInstance): - """Test `cloud-init schema` with valid userdata. - - PR #575 - """ - result = client.execute("cloud-init schema --system") - assert result.ok - assert "Valid cloud-config: system userdata" == result.stdout.strip() - result = client.execute("cloud-init status --long") - if not result.ok: - raise AssertionError( - f"Unexpected error from cloud-init status: {result}" - ) - - -@pytest.mark.user_data(INVALID_USER_DATA_HEADER) -def test_invalid_userdata(client: IntegrationInstance): - """Test `cloud-init schema` with invalid userdata. - - PR #575 - """ - result = client.execute("cloud-init schema --system") - assert not result.ok - assert "Cloud config schema errors" in result.stderr - assert 'needs to begin with "#cloud-config"' in result.stderr - result = client.execute("cloud-init status --long") - if not result.ok: - raise AssertionError( - f"Unexpected error from cloud-init status: {result}" - ) - - -@pytest.mark.user_data(INVALID_USER_DATA_SCHEMA) -def test_invalid_userdata_schema(client: IntegrationInstance): - """Test invalid schema represented as Warnings, not fatal - - PR #1175 - """ - result = client.execute("cloud-init status --long") - assert result.ok - log = client.read_from_file("/var/log/cloud-init.log") - warning = ( - "[WARNING]: Invalid cloud-config provided:\napt_pipelining: 'bogus'" - " is not valid under any of the given schemas\nupdates: Additional" - " properties are not allowed ('notnetwork' was unexpected)" - ) - assert warning in log - result = client.execute("cloud-init status --long") - if not result.ok: - raise AssertionError( - f"Unexpected error from cloud-init status: {result}" - ) diff --git a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/tests/unittests/config/test_schema.py b/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/tests/unittests/config/test_schema.py deleted file mode 100644 index c75b72270..000000000 --- a/.pc/cpick-b0534cbf-Remove-schema-errors-from-log/tests/unittests/config/test_schema.py +++ /dev/null @@ -1,1124 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - - -import importlib -import inspect -import itertools -import json -import logging -import os -import sys -from copy import copy, deepcopy -from pathlib import Path -from textwrap import dedent -from types import ModuleType -from typing import List - -import jsonschema -import pytest - -from cloudinit.config.schema import ( - CLOUD_CONFIG_HEADER, - VERSIONED_USERDATA_SCHEMA_FILE, - MetaSchema, - SchemaValidationError, - annotated_cloudconfig_file, - get_jsonschema_validator, - get_meta_doc, - get_schema, - get_schema_dir, - load_doc, - main, - validate_cloudconfig_file, - validate_cloudconfig_metaschema, - validate_cloudconfig_schema, -) -from cloudinit.distros import OSFAMILIES -from cloudinit.safeyaml import load, load_with_marks -from cloudinit.settings import FREQUENCIES -from cloudinit.util import load_file, write_file -from tests.unittests.helpers import ( - CiTestCase, - cloud_init_project_dir, - mock, - skipUnlessJsonSchema, -) - - -def get_schemas() -> dict: - """Return all legacy module schemas - - Assumes that module schemas have the variable name "schema" - """ - return get_module_variable("schema") - - -def get_metas() -> dict: - """Return all module metas - - Assumes that module schemas have the variable name "schema" - """ - return get_module_variable("meta") - - -def get_module_names() -> List[str]: - """Return list of module names in cloudinit/config""" - files = list( - Path(cloud_init_project_dir("cloudinit/config/")).glob("cc_*.py") - ) - - return [mod.stem for mod in files] - - -def get_modules() -> List[ModuleType]: - """Return list of modules in cloudinit/config""" - return [ - importlib.import_module(f"cloudinit.config.{module}") - for module in get_module_names() - ] - - -def get_module_variable(var_name) -> dict: - """Inspect modules and get variable from module matching var_name""" - schemas = {} - get_modules() - for k, v in sys.modules.items(): - path = Path(k) - if "cloudinit.config" == path.stem and path.suffix[1:4] == "cc_": - module_name = path.suffix[1:] - members = inspect.getmembers(v) - schemas[module_name] = None - for name, value in members: - if name == var_name: - schemas[module_name] = value - break - return schemas - - -class TestVersionedSchemas: - def _relative_ref_to_local_file_path(self, source_schema): - """Replace known relative ref URLs with full file path.""" - # jsonschema 2.6.0 doesn't support relative URLs in $refs (bionic) - full_path_schema = deepcopy(source_schema) - relative_ref = full_path_schema["oneOf"][0]["allOf"][1]["$ref"] - full_local_filepath = get_schema_dir() + relative_ref[1:] - file_ref = f"file://{full_local_filepath}" - full_path_schema["oneOf"][0]["allOf"][1]["$ref"] = file_ref - return full_path_schema - - @pytest.mark.parametrize( - "schema,error_msg", - ( - ({}, None), - ({"version": "v1"}, None), - ({"version": "v2"}, "is not valid"), - ({"version": "v1", "final_message": -1}, "is not valid"), - ({"version": "v1", "final_message": "some msg"}, None), - ), - ) - def test_versioned_cloud_config_schema_is_valid_json( - self, schema, error_msg - ): - version_schemafile = os.path.join( - get_schema_dir(), VERSIONED_USERDATA_SCHEMA_FILE - ) - version_schema = json.loads(load_file(version_schemafile)) - # To avoid JSON resolver trying to pull the reference from our - # upstream raw file in github. - version_schema["$id"] = f"file://{version_schemafile}" - if error_msg: - with pytest.raises(SchemaValidationError) as context_mgr: - try: - validate_cloudconfig_schema( - schema, schema=version_schema, strict=True - ) - except jsonschema.exceptions.RefResolutionError: - full_path_schema = self._relative_ref_to_local_file_path( - version_schema - ) - validate_cloudconfig_schema( - schema, schema=full_path_schema, strict=True - ) - assert error_msg in str(context_mgr.value) - else: - try: - validate_cloudconfig_schema( - schema, schema=version_schema, strict=True - ) - except jsonschema.exceptions.RefResolutionError: - full_path_schema = self._relative_ref_to_local_file_path( - version_schema - ) - validate_cloudconfig_schema( - schema, schema=full_path_schema, strict=True - ) - - -class TestGetSchema: - def test_static_schema_file_is_valid(self, caplog): - with caplog.at_level(logging.WARNING): - get_schema() - # Assert no warnings parsing our packaged schema file - warnings = [msg for (_, _, msg) in caplog.record_tuples] - assert [] == warnings - - def test_get_schema_coalesces_known_schema(self): - """Every cloudconfig module with schema is listed in allOf keyword.""" - schema = get_schema() - assert sorted(get_module_names()) == sorted( - [meta["id"] for meta in get_metas().values() if meta is not None] - ) - assert "http://json-schema.org/draft-04/schema#" == schema["$schema"] - assert ["$defs", "$schema", "allOf"] == sorted(list(schema.keys())) - # New style schema should be defined in static schema file in $defs - expected_subschema_defs = [ - {"$ref": "#/$defs/cc_apk_configure"}, - {"$ref": "#/$defs/cc_apt_configure"}, - {"$ref": "#/$defs/cc_apt_pipelining"}, - {"$ref": "#/$defs/cc_bootcmd"}, - {"$ref": "#/$defs/cc_byobu"}, - {"$ref": "#/$defs/cc_ca_certs"}, - {"$ref": "#/$defs/cc_chef"}, - {"$ref": "#/$defs/cc_debug"}, - {"$ref": "#/$defs/cc_disable_ec2_metadata"}, - {"$ref": "#/$defs/cc_disk_setup"}, - {"$ref": "#/$defs/cc_fan"}, - {"$ref": "#/$defs/cc_final_message"}, - {"$ref": "#/$defs/cc_growpart"}, - {"$ref": "#/$defs/cc_grub_dpkg"}, - {"$ref": "#/$defs/cc_install_hotplug"}, - {"$ref": "#/$defs/cc_keyboard"}, - {"$ref": "#/$defs/cc_keys_to_console"}, - {"$ref": "#/$defs/cc_landscape"}, - {"$ref": "#/$defs/cc_locale"}, - {"$ref": "#/$defs/cc_lxd"}, - {"$ref": "#/$defs/cc_mcollective"}, - {"$ref": "#/$defs/cc_migrator"}, - {"$ref": "#/$defs/cc_mounts"}, - {"$ref": "#/$defs/cc_ntp"}, - {"$ref": "#/$defs/cc_package_update_upgrade_install"}, - {"$ref": "#/$defs/cc_phone_home"}, - {"$ref": "#/$defs/cc_power_state_change"}, - {"$ref": "#/$defs/cc_puppet"}, - {"$ref": "#/$defs/cc_resizefs"}, - {"$ref": "#/$defs/cc_resolv_conf"}, - {"$ref": "#/$defs/cc_rh_subscription"}, - {"$ref": "#/$defs/cc_rsyslog"}, - {"$ref": "#/$defs/cc_runcmd"}, - {"$ref": "#/$defs/cc_salt_minion"}, - {"$ref": "#/$defs/cc_scripts_vendor"}, - {"$ref": "#/$defs/cc_seed_random"}, - {"$ref": "#/$defs/cc_set_hostname"}, - {"$ref": "#/$defs/cc_set_passwords"}, - {"$ref": "#/$defs/cc_snap"}, - {"$ref": "#/$defs/cc_spacewalk"}, - {"$ref": "#/$defs/cc_ssh_authkey_fingerprints"}, - {"$ref": "#/$defs/cc_ssh_import_id"}, - {"$ref": "#/$defs/cc_ssh"}, - {"$ref": "#/$defs/cc_timezone"}, - {"$ref": "#/$defs/cc_ubuntu_advantage"}, - {"$ref": "#/$defs/cc_ubuntu_drivers"}, - {"$ref": "#/$defs/cc_update_etc_hosts"}, - {"$ref": "#/$defs/cc_update_hostname"}, - {"$ref": "#/$defs/cc_users_groups"}, - {"$ref": "#/$defs/cc_write_files"}, - {"$ref": "#/$defs/cc_yum_add_repo"}, - {"$ref": "#/$defs/cc_zypper_add_repo"}, - ] - found_subschema_defs = [] - legacy_schema_keys = [] - for subschema in schema["allOf"]: - if "$ref" in subschema: - found_subschema_defs.append(subschema) - else: # Legacy subschema sourced from cc_* module 'schema' attr - legacy_schema_keys.extend(subschema["properties"].keys()) - - assert expected_subschema_defs == found_subschema_defs - # This list should remain empty unless we induct new modules with - # legacy schema attributes defined within the cc_module. - assert [] == sorted(legacy_schema_keys) - - -class TestLoadDoc: - - docs = get_module_variable("__doc__") - - @pytest.mark.parametrize( - "module_name", - ("cc_apt_pipelining",), # new style composite schema file - ) - def test_report_docs_consolidated_schema(self, module_name): - doc = load_doc([module_name]) - assert doc, "Unexpected empty docs for {}".format(module_name) - assert self.docs[module_name] == doc - - -class SchemaValidationErrorTest(CiTestCase): - """Test validate_cloudconfig_schema""" - - def test_schema_validation_error_expects_schema_errors(self): - """SchemaValidationError is initialized from schema_errors.""" - errors = ( - ("key.path", 'unexpected key "junk"'), - ("key2.path", '"-123" is not a valid "hostname" format'), - ) - exception = SchemaValidationError(schema_errors=errors) - self.assertIsInstance(exception, Exception) - self.assertEqual(exception.schema_errors, errors) - self.assertEqual( - 'Cloud config schema errors: key.path: unexpected key "junk", ' - 'key2.path: "-123" is not a valid "hostname" format', - str(exception), - ) - self.assertTrue(isinstance(exception, ValueError)) - - -class TestValidateCloudConfigSchema: - """Tests for validate_cloudconfig_schema.""" - - with_logs = True - - @pytest.mark.parametrize( - "schema, call_count", - ((None, 1), ({"properties": {"p1": {"type": "string"}}}, 0)), - ) - @skipUnlessJsonSchema() - @mock.patch("cloudinit.config.schema.get_schema") - def test_validateconfig_schema_use_full_schema_when_no_schema_param( - self, get_schema, schema, call_count - ): - """Use full schema when schema param is absent.""" - get_schema.return_value = {"properties": {"p1": {"type": "string"}}} - kwargs = {"config": {"p1": "valid"}} - if schema: - kwargs["schema"] = schema - validate_cloudconfig_schema(**kwargs) - assert call_count == get_schema.call_count - - @skipUnlessJsonSchema() - def test_validateconfig_schema_non_strict_emits_warnings(self, caplog): - """When strict is False validate_cloudconfig_schema emits warnings.""" - schema = {"properties": {"p1": {"type": "string"}}} - validate_cloudconfig_schema({"p1": -1}, schema, strict=False) - [(module, log_level, log_msg)] = caplog.record_tuples - assert "cloudinit.config.schema" == module - assert logging.WARNING == log_level - assert ( - "Invalid cloud-config provided:\np1: -1 is not of type 'string'" - == log_msg - ) - - @skipUnlessJsonSchema() - def test_validateconfig_schema_emits_warning_on_missing_jsonschema( - self, caplog - ): - """Warning from validate_cloudconfig_schema when missing jsonschema.""" - schema = {"properties": {"p1": {"type": "string"}}} - with mock.patch.dict("sys.modules", **{"jsonschema": ImportError()}): - validate_cloudconfig_schema({"p1": -1}, schema, strict=True) - assert "Ignoring schema validation. jsonschema is not present" in ( - caplog.text - ) - - @skipUnlessJsonSchema() - def test_validateconfig_schema_strict_raises_errors(self): - """When strict is True validate_cloudconfig_schema raises errors.""" - schema = {"properties": {"p1": {"type": "string"}}} - with pytest.raises(SchemaValidationError) as context_mgr: - validate_cloudconfig_schema({"p1": -1}, schema, strict=True) - assert ( - "Cloud config schema errors: p1: -1 is not of type 'string'" - == (str(context_mgr.value)) - ) - - @skipUnlessJsonSchema() - def test_validateconfig_schema_honors_formats(self): - """With strict True, validate_cloudconfig_schema errors on format.""" - schema = {"properties": {"p1": {"type": "string", "format": "email"}}} - with pytest.raises(SchemaValidationError) as context_mgr: - validate_cloudconfig_schema({"p1": "-1"}, schema, strict=True) - assert "Cloud config schema errors: p1: '-1' is not a 'email'" == ( - str(context_mgr.value) - ) - - @skipUnlessJsonSchema() - def test_validateconfig_schema_honors_formats_strict_metaschema(self): - """With strict and strict_metaschema True, ensure errors on format""" - schema = {"properties": {"p1": {"type": "string", "format": "email"}}} - with pytest.raises(SchemaValidationError) as context_mgr: - validate_cloudconfig_schema( - {"p1": "-1"}, schema, strict=True, strict_metaschema=True - ) - assert "Cloud config schema errors: p1: '-1' is not a 'email'" == str( - context_mgr.value - ) - - @skipUnlessJsonSchema() - def test_validateconfig_strict_metaschema_do_not_raise_exception( - self, caplog - ): - """With strict_metaschema=True, do not raise exceptions. - - This flag is currently unused, but is intended for run-time validation. - This should warn, but not raise. - """ - schema = {"properties": {"p1": {"types": "string", "format": "email"}}} - validate_cloudconfig_schema( - {"p1": "-1"}, schema, strict_metaschema=True - ) - assert ( - "Meta-schema validation failed, attempting to validate config" - in caplog.text - ) - - -class TestCloudConfigExamples: - metas = get_metas() - params = [ - (meta["id"], example) - for meta in metas.values() - if meta and meta.get("examples") - for example in meta.get("examples") - ] - - @pytest.mark.parametrize("schema_id, example", params) - @skipUnlessJsonSchema() - def test_validateconfig_schema_of_example(self, schema_id, example): - """For a given example in a config module we test if it is valid - according to the unified schema of all config modules - """ - schema = get_schema() - config_load = load(example) - # cloud-init-schema-v1 is permissive of additionalProperties at the - # top-level. - # To validate specific schemas against known documented examples - # we need to only define the specific module schema and supply - # strict=True. - # TODO(Drop to pop/update once full schema is strict) - schema.pop("allOf") - schema.update(schema["$defs"][schema_id]) - schema["additionalProperties"] = False - # Some module examples reference keys defined in multiple schemas - supplemental_schemas = { - "cc_ubuntu_advantage": ["cc_power_state_change"], - "cc_update_hostname": ["cc_set_hostname"], - "cc_users_groups": ["cc_ssh_import_id"], - "cc_disk_setup": ["cc_mounts"], - } - for supplement_id in supplemental_schemas.get(schema_id, []): - supplemental_props = dict( - [ - (key, value) - for key, value in schema["$defs"][supplement_id][ - "properties" - ].items() - ] - ) - schema["properties"].update(supplemental_props) - validate_cloudconfig_schema(config_load, schema, strict=True) - - -class TestValidateCloudConfigFile: - """Tests for validate_cloudconfig_file.""" - - @pytest.mark.parametrize("annotate", (True, False)) - def test_validateconfig_file_error_on_absent_file(self, annotate): - """On absent config_path, validate_cloudconfig_file errors.""" - with pytest.raises( - RuntimeError, match="Configfile /not/here does not exist" - ): - validate_cloudconfig_file("/not/here", {}, annotate) - - @pytest.mark.parametrize("annotate", (True, False)) - def test_validateconfig_file_error_on_invalid_header( - self, annotate, tmpdir - ): - """On invalid header, validate_cloudconfig_file errors. - - A SchemaValidationError is raised when the file doesn't begin with - CLOUD_CONFIG_HEADER. - """ - config_file = tmpdir.join("my.yaml") - config_file.write("#junk") - error_msg = ( - "Cloud config schema errors: format-l1.c1: File" - f" {config_file} needs to begin with" - f' "{CLOUD_CONFIG_HEADER.decode()}"' - ) - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_file(config_file.strpath, {}, annotate) - - @pytest.mark.parametrize("annotate", (True, False)) - def test_validateconfig_file_error_on_non_yaml_scanner_error( - self, annotate, tmpdir - ): - """On non-yaml scan issues, validate_cloudconfig_file errors.""" - # Generate a scanner error by providing text on a single line with - # improper indent. - config_file = tmpdir.join("my.yaml") - config_file.write("#cloud-config\nasdf:\nasdf") - error_msg = ( - f".*errors: format-l3.c1: File {config_file} is not valid yaml.*" - ) - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_file(config_file.strpath, {}, annotate) - - @pytest.mark.parametrize("annotate", (True, False)) - def test_validateconfig_file_error_on_non_yaml_parser_error( - self, annotate, tmpdir - ): - """On non-yaml parser issues, validate_cloudconfig_file errors.""" - config_file = tmpdir.join("my.yaml") - config_file.write("#cloud-config\n{}}") - error_msg = ( - f"errors: format-l2.c3: File {config_file} is not valid yaml." - ) - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_file(config_file.strpath, {}, annotate) - - @skipUnlessJsonSchema() - @pytest.mark.parametrize("annotate", (True, False)) - def test_validateconfig_file_sctrictly_validates_schema( - self, annotate, tmpdir - ): - """validate_cloudconfig_file raises errors on invalid schema.""" - config_file = tmpdir.join("my.yaml") - schema = {"properties": {"p1": {"type": "string", "format": "string"}}} - config_file.write("#cloud-config\np1: -1") - error_msg = ( - "Cloud config schema errors: p1: -1 is not of type 'string'" - ) - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_file(config_file.strpath, schema, annotate) - - -class TestSchemaDocMarkdown: - """Tests for get_meta_doc.""" - - required_schema = { - "title": "title", - "description": "description", - "id": "id", - "name": "name", - "frequency": "frequency", - "distros": ["debian", "rhel"], - } - meta: MetaSchema = { - "title": "title", - "description": "description", - "id": "id", - "name": "name", - "frequency": "frequency", - "distros": ["debian", "rhel"], - "examples": [ - 'ex1:\n [don\'t, expand, "this"]', - "ex2: true", - ], - } - - def test_get_meta_doc_returns_restructured_text(self): - """get_meta_doc returns restructured text for a cloudinit schema.""" - full_schema = copy(self.required_schema) - full_schema.update( - { - "properties": { - "prop1": { - "type": "array", - "description": "prop-description", - "items": {"type": "integer"}, - } - } - } - ) - - doc = get_meta_doc(self.meta, full_schema) - assert ( - dedent( - """ - name - ---- - **Summary:** title - - description - - **Internal name:** ``id`` - - **Module frequency:** frequency - - **Supported distros:** debian, rhel - - **Config schema**: - **prop1:** (array of integer) prop-description - - **Examples**:: - - ex1: - [don't, expand, "this"] - # --- Example2 --- - ex2: true - """ - ) - == doc - ) - - def test_get_meta_doc_handles_multiple_types(self): - """get_meta_doc delimits multiple property types with a '/'.""" - schema = {"properties": {"prop1": {"type": ["string", "integer"]}}} - assert "**prop1:** (string/integer)" in get_meta_doc(self.meta, schema) - - def test_references_are_flattened_in_schema_docs(self): - """get_meta_doc flattens and renders full schema definitions.""" - schema = { - "$defs": { - "flattenit": { - "type": ["object", "string"], - "description": "Objects support the following keys:", - "patternProperties": { - "^.+$": { - "label": "", - "description": "List of cool strings", - "type": "array", - "items": {"type": "string"}, - "minItems": 1, - } - }, - } - }, - "properties": {"prop1": {"$ref": "#/$defs/flattenit"}}, - } - assert ( - dedent( - """\ - **prop1:** (string/object) Objects support the following keys: - - **:** (array of string) List of cool strings - """ - ) - in get_meta_doc(self.meta, schema) - ) - - @pytest.mark.parametrize( - "sub_schema,expected", - ( - ( - {"enum": [True, False, "stuff"]}, - "**prop1:** (``true``/``false``/``stuff``)", - ), - # When type: string and enum, document enum values - ( - {"type": "string", "enum": ["a", "b"]}, - "**prop1:** (``a``/``b``)", - ), - ), - ) - def test_get_meta_doc_handles_enum_types(self, sub_schema, expected): - """get_meta_doc converts enum types to yaml and delimits with '/'.""" - schema = {"properties": {"prop1": sub_schema}} - assert expected in get_meta_doc(self.meta, schema) - - @pytest.mark.parametrize( - "schema,expected", - ( - ( # Hide top-level keys like 'properties' - { - "hidden": ["properties"], - "properties": { - "p1": {"type": "string"}, - "p2": {"type": "boolean"}, - }, - "patternProperties": { - "^.*$": { - "type": "string", - "label": "label2", - } - }, - }, - dedent( - """ - **Config schema**: - **label2:** (string) - """ - ), - ), - ( # Hide nested individual keys with a bool - { - "properties": { - "p1": {"type": "string", "hidden": True}, - "p2": {"type": "boolean"}, - } - }, - dedent( - """ - **Config schema**: - **p2:** (boolean) - """ - ), - ), - ), - ) - def test_get_meta_doc_hidden_hides_specific_properties_from_docs( - self, schema, expected - ): - """Docs are hidden for any property in the hidden list. - - Useful for hiding deprecated key schema. - """ - assert expected in get_meta_doc(self.meta, schema) - - def test_get_meta_doc_handles_nested_oneof_property_types(self): - """get_meta_doc describes array items oneOf declarations in type.""" - schema = { - "properties": { - "prop1": { - "type": "array", - "items": { - "oneOf": [{"type": "string"}, {"type": "integer"}] - }, - } - } - } - assert "**prop1:** (array of (string/integer))" in get_meta_doc( - self.meta, schema - ) - - def test_get_meta_doc_handles_types_as_list(self): - """get_meta_doc renders types which have a list value.""" - schema = { - "properties": { - "prop1": { - "type": ["boolean", "array"], - "items": { - "oneOf": [{"type": "string"}, {"type": "integer"}] - }, - } - } - } - assert ( - "**prop1:** (boolean/array of (string/integer))" - in get_meta_doc(self.meta, schema) - ) - - def test_get_meta_doc_handles_flattening_defs(self): - """get_meta_doc renders $defs.""" - schema = { - "$defs": { - "prop1object": { - "type": "object", - "properties": {"subprop": {"type": "string"}}, - } - }, - "properties": {"prop1": {"$ref": "#/$defs/prop1object"}}, - } - assert ( - "**prop1:** (object)\n\n **subprop:** (string)\n" - in get_meta_doc(self.meta, schema) - ) - - def test_get_meta_doc_handles_string_examples(self): - """get_meta_doc properly indented examples as a list of strings.""" - full_schema = copy(self.required_schema) - full_schema.update( - { - "examples": [ - 'ex1:\n [don\'t, expand, "this"]', - "ex2: true", - ], - "properties": { - "prop1": { - "type": "array", - "description": "prop-description", - "items": {"type": "integer"}, - } - }, - } - ) - assert ( - dedent( - """ - **Config schema**: - **prop1:** (array of integer) prop-description - - **Examples**:: - - ex1: - [don't, expand, "this"] - # --- Example2 --- - ex2: true - """ - ) - in get_meta_doc(self.meta, full_schema) - ) - - def test_get_meta_doc_properly_parse_description(self): - """get_meta_doc description properly formatted""" - schema = { - "properties": { - "p1": { - "type": "string", - "description": dedent( - """\ - This item - has the - following options: - - - option1 - - option2 - - option3 - - The default value is - option1""" - ), - } - } - } - - assert ( - dedent( - """ - **Config schema**: - **p1:** (string) This item has the following options: - - - option1 - - option2 - - option3 - - The default value is option1 - - """ - ) - in get_meta_doc(self.meta, schema) - ) - - def test_get_meta_doc_raises_key_errors(self): - """get_meta_doc raises KeyErrors on missing keys.""" - schema = { - "properties": { - "prop1": { - "type": "array", - "items": { - "oneOf": [{"type": "string"}, {"type": "integer"}] - }, - } - } - } - for key in self.meta: - invalid_meta = copy(self.meta) - invalid_meta.pop(key) - with pytest.raises(KeyError) as context_mgr: - get_meta_doc(invalid_meta, schema) - assert key in str(context_mgr.value) - - def test_label_overrides_property_name(self): - """get_meta_doc overrides property name with label.""" - schema = { - "properties": { - "prop1": { - "type": "string", - "label": "label1", - }, - "prop_no_label": { - "type": "string", - }, - "prop_array": { - "label": "array_label", - "type": "array", - "items": { - "type": "object", - "properties": { - "some_prop": {"type": "number"}, - }, - }, - }, - }, - "patternProperties": { - "^.*$": { - "type": "string", - "label": "label2", - } - }, - } - meta_doc = get_meta_doc(self.meta, schema) - assert "**label1:** (string)" in meta_doc - assert "**label2:** (string" in meta_doc - assert "**prop_no_label:** (string)" in meta_doc - assert "Each object in **array_label** list" in meta_doc - - assert "prop1" not in meta_doc - assert ".*" not in meta_doc - - -class TestAnnotatedCloudconfigFile: - def test_annotated_cloudconfig_file_no_schema_errors(self): - """With no schema_errors, print the original content.""" - content = b"ntp:\n pools: [ntp1.pools.com]\n" - parse_cfg, schemamarks = load_with_marks(content) - assert content == annotated_cloudconfig_file( - parse_cfg, content, schema_errors=[], schemamarks=schemamarks - ) - - def test_annotated_cloudconfig_file_with_non_dict_cloud_config(self): - """Error when empty non-dict cloud-config is provided. - - OurJSON validation when user-data is None type generates a bunch - schema validation errors of the format: - ('', "None is not of type 'object'"). Ignore those symptoms and - report the general problem instead. - """ - content = b"\n\n\n" - expected = "\n".join( - [ - content.decode(), - "# Errors: -------------", - "# E1: Cloud-config is not a YAML dict.\n\n", - ] - ) - assert expected == annotated_cloudconfig_file( - None, - content, - schema_errors=[("", "None is not of type 'object'")], - schemamarks={}, - ) - - def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self): - """With schema_errors, error lines are annotated and a footer added.""" - content = dedent( - """\ - #cloud-config - # comment - ntp: - pools: [-99, 75] - """ - ).encode() - expected = dedent( - """\ - #cloud-config - # comment - ntp: # E1 - pools: [-99, 75] # E2,E3 - - # Errors: ------------- - # E1: Some type error - # E2: -99 is not a string - # E3: 75 is not a string - - """ - ) - parsed_config, schemamarks = load_with_marks(content[13:]) - schema_errors = [ - ("ntp", "Some type error"), - ("ntp.pools.0", "-99 is not a string"), - ("ntp.pools.1", "75 is not a string"), - ] - assert expected == annotated_cloudconfig_file( - parsed_config, content, schema_errors, schemamarks=schemamarks - ) - - def test_annotated_cloudconfig_file_annotates_separate_line_items(self): - """Errors are annotated for lists with items on separate lines.""" - content = dedent( - """\ - #cloud-config - # comment - ntp: - pools: - - -99 - - 75 - """ - ).encode() - expected = dedent( - """\ - ntp: - pools: - - -99 # E1 - - 75 # E2 - """ - ) - parsed_config, schemamarks = load_with_marks(content[13:]) - schema_errors = [ - ("ntp.pools.0", "-99 is not a string"), - ("ntp.pools.1", "75 is not a string"), - ] - assert expected in annotated_cloudconfig_file( - parsed_config, content, schema_errors, schemamarks=schemamarks - ) - - -class TestMain: - - exclusive_combinations = itertools.combinations( - ["--system", "--docs all", "--config-file something"], 2 - ) - - @pytest.mark.parametrize("params", exclusive_combinations) - def test_main_exclusive_args(self, params, capsys): - """Main exits non-zero and error on required exclusive args.""" - params = list(itertools.chain(*[a.split() for a in params])) - with mock.patch("sys.argv", ["mycmd"] + params): - with pytest.raises(SystemExit) as context_manager: - main() - assert 1 == context_manager.value.code - - _out, err = capsys.readouterr() - expected = ( - "Error:\n" - "Expected one of --config-file, --system or --docs arguments\n" - ) - assert expected == err - - def test_main_missing_args(self, capsys): - """Main exits non-zero and reports an error on missing parameters.""" - with mock.patch("sys.argv", ["mycmd"]): - with pytest.raises(SystemExit) as context_manager: - main() - assert 1 == context_manager.value.code - - _out, err = capsys.readouterr() - expected = ( - "Error:\n" - "Expected one of --config-file, --system or --docs arguments\n" - ) - assert expected == err - - def test_main_absent_config_file(self, capsys): - """Main exits non-zero when config file is absent.""" - myargs = ["mycmd", "--annotate", "--config-file", "NOT_A_FILE"] - with mock.patch("sys.argv", myargs): - with pytest.raises(SystemExit) as context_manager: - main() - assert 1 == context_manager.value.code - _out, err = capsys.readouterr() - assert "Error:\nConfigfile NOT_A_FILE does not exist\n" == err - - def test_main_invalid_flag_combo(self, capsys): - """Main exits non-zero when invalid flag combo used.""" - myargs = ["mycmd", "--annotate", "--docs", "DOES_NOT_MATTER"] - with mock.patch("sys.argv", myargs): - with pytest.raises(SystemExit) as context_manager: - main() - assert 1 == context_manager.value.code - _, err = capsys.readouterr() - assert ( - "Error:\nInvalid flag combination. " - "Cannot use --annotate with --docs\n" == err - ) - - def test_main_prints_docs(self, capsys): - """When --docs parameter is provided, main generates documentation.""" - myargs = ["mycmd", "--docs", "all"] - with mock.patch("sys.argv", myargs): - assert 0 == main(), "Expected 0 exit code" - out, _err = capsys.readouterr() - assert "\nNTP\n---\n" in out - assert "\nRuncmd\n------\n" in out - - def test_main_validates_config_file(self, tmpdir, capsys): - """When --config-file parameter is provided, main validates schema.""" - myyaml = tmpdir.join("my.yaml") - myargs = ["mycmd", "--config-file", myyaml.strpath] - myyaml.write(b"#cloud-config\nntp:") # shortest ntp schema - with mock.patch("sys.argv", myargs): - assert 0 == main(), "Expected 0 exit code" - out, _err = capsys.readouterr() - assert "Valid cloud-config: {0}\n".format(myyaml) == out - - @mock.patch("cloudinit.config.schema.read_cfg_paths") - @mock.patch("cloudinit.config.schema.os.getuid", return_value=0) - def test_main_validates_system_userdata( - self, m_getuid, m_read_cfg_paths, capsys, paths - ): - """When --system is provided, main validates system userdata.""" - m_read_cfg_paths.return_value = paths - ud_file = paths.get_ipath_cur("userdata_raw") - write_file(ud_file, b"#cloud-config\nntp:") - myargs = ["mycmd", "--system"] - with mock.patch("sys.argv", myargs): - assert 0 == main(), "Expected 0 exit code" - out, _err = capsys.readouterr() - assert "Valid cloud-config: system userdata\n" == out - - @mock.patch("cloudinit.config.schema.os.getuid", return_value=1000) - def test_main_system_userdata_requires_root(self, m_getuid, capsys, paths): - """Non-root user can't use --system param""" - myargs = ["mycmd", "--system"] - with mock.patch("sys.argv", myargs): - with pytest.raises(SystemExit) as context_manager: - main() - assert 1 == context_manager.value.code - _out, err = capsys.readouterr() - expected = ( - "Error:\nUnable to read system userdata as non-root user. " - "Try using sudo\n" - ) - assert expected == err - - -def _get_meta_doc_examples(): - examples_dir = Path(cloud_init_project_dir("doc/examples")) - assert examples_dir.is_dir() - - return ( - str(f) - for f in examples_dir.glob("cloud-config*.txt") - if not f.name.startswith("cloud-config-archive") - ) - - -class TestSchemaDocExamples: - schema = get_schema() - - @pytest.mark.parametrize("example_path", _get_meta_doc_examples()) - @skipUnlessJsonSchema() - def test_schema_doc_examples(self, example_path): - validate_cloudconfig_file(example_path, self.schema) - - -class TestStrictMetaschema: - """Validate that schemas follow a stricter metaschema definition than - the default. This disallows arbitrary key/value pairs. - """ - - @skipUnlessJsonSchema() - def test_modules(self): - """Validate all modules with a stricter metaschema""" - (validator, _) = get_jsonschema_validator() - for (name, value) in get_schemas().items(): - if value: - validate_cloudconfig_metaschema(validator, value) - else: - logging.warning("module %s has no schema definition", name) - - @skipUnlessJsonSchema() - def test_validate_bad_module(self): - """Throw exception by default, don't throw if throw=False - - item should be 'items' and is therefore interpreted as an additional - property which is invalid with a strict metaschema - """ - (validator, _) = get_jsonschema_validator() - schema = { - "type": "array", - "item": { - "type": "object", - }, - } - with pytest.raises( - SchemaValidationError, - match=r"Additional properties are not allowed.*", - ): - - validate_cloudconfig_metaschema(validator, schema) - - validate_cloudconfig_metaschema(validator, schema, throw=False) - - -class TestMeta: - def test_valid_meta_for_every_module(self): - all_distros = { - name for distro in OSFAMILIES.values() for name in distro - } - all_distros.add("all") - for module in get_modules(): - assert "frequency" in module.meta - assert "distros" in module.meta - assert {module.meta["frequency"]}.issubset(FREQUENCIES) - assert set(module.meta["distros"]).issubset(all_distros) diff --git a/.pc/expire-on-hashed-users.patch/cloudinit/features.py b/.pc/expire-on-hashed-users.patch/cloudinit/features.py new file mode 100644 index 000000000..ac586f6b3 --- /dev/null +++ b/.pc/expire-on-hashed-users.patch/cloudinit/features.py @@ -0,0 +1,66 @@ +# This file is part of cloud-init. See LICENSE file for license information. +""" +Feature flags are used as a way to easily toggle configuration +**at build time**. They are provided to accommodate feature deprecation and +downstream configuration changes. + +Currently used upstream values for feature flags are set in +``cloudinit/features.py``. Overrides to these values (typically via quilt +patch) can be placed +in a file called ``feature_overrides.py`` in the same directory. Any value +set in ``feature_overrides.py`` will override the original value set +in ``features.py``. + +Each flag should include a short comment regarding the reason for +the flag and intended lifetime. + +Tests are required for new feature flags, and tests must verify +all valid states of a flag, not just the default state. +""" + +ERROR_ON_USER_DATA_FAILURE = True +""" +If there is a failure in obtaining user data (i.e., #include or +decompress fails) and ``ERROR_ON_USER_DATA_FAILURE`` is ``False``, +cloud-init will log a warning and proceed. If it is ``True``, +cloud-init will instead raise an exception. + +As of 20.3, ``ERROR_ON_USER_DATA_FAILURE`` is ``True``. + +(This flag can be removed after Focal is no longer supported.) +""" + + +ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES = False +""" +When configuring apt mirrors, if +``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``True`` cloud-init +will detect that a datasource's ``availability_zone`` property looks +like an EC2 availability zone and set the ``ec2_region`` variable when +generating mirror URLs; this can lead to incorrect mirrors being +configured in clouds whose AZs follow EC2's naming pattern. + +As of 20.3, ``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``False`` +so we no longer include ``ec2_region`` in mirror determination on +non-AWS cloud platforms. + +If the old behavior is desired, users can provide the appropriate +mirrors via :py:mod:`apt: ` +directives in cloud-config. +""" + + +EXPIRE_APPLIES_TO_HASHED_USERS = True +""" +If ``EXPIRE_APPLIES_TO_HASHED_USERS`` is True, then when expire is set true +in cc_set_passwords, hashed passwords will be expired. Previous to 22.3, +only non-hashed passwords were expired. + +(This flag can be removed after Jammy is no longer supported.) +""" + +try: + # pylint: disable=wildcard-import + from cloudinit.feature_overrides import * # noqa +except ImportError: + pass diff --git a/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py b/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py new file mode 100644 index 000000000..10473c3bd --- /dev/null +++ b/.pc/expire-on-hashed-users.patch/tests/unittests/config/test_cc_set_passwords.py @@ -0,0 +1,872 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import logging +from unittest import mock + +import pytest + +from cloudinit import features, subp, util +from cloudinit.config import cc_set_passwords as setpass +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import does_not_raise, skipUnlessJsonSchema +from tests.unittests.util import get_cloud + +MODPATH = "cloudinit.config.cc_set_passwords." +LOG = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True) +def common_fixtures(mocker): + mocker.patch("cloudinit.distros.uses_systemd", return_value=True) + mocker.patch("cloudinit.util.write_to_console") + + +class TestHandleSSHPwauth: + @pytest.mark.parametrize( + "uses_systemd,cmd", + ( + (True, ["systemctl", "status", "ssh"]), + (False, ["service", "ssh", "status"]), + ), + ) + @mock.patch("cloudinit.distros.subp.subp") + def test_unknown_value_logs_warning( + self, m_subp, uses_systemd, cmd, caplog + ): + cloud = get_cloud("ubuntu") + with mock.patch.object( + cloud.distro, "uses_systemd", return_value=uses_systemd + ): + setpass.handle_ssh_pwauth("floo", cloud.distro) + assert "Unrecognized value: ssh_pwauth=floo" in caplog.text + assert [mock.call(cmd, capture=True)] == m_subp.call_args_list + + @pytest.mark.parametrize( + "uses_systemd,ssh_updated,cmd,expected_log", + ( + ( + True, + True, + ["systemctl", "restart", "ssh"], + "Restarted the SSH daemon.", + ), + ( + True, + False, + ["systemctl", "status", "ssh"], + "No need to restart SSH", + ), + ( + False, + True, + ["service", "ssh", "restart"], + "Restarted the SSH daemon.", + ), + ( + False, + False, + ["service", "ssh", "status"], + "No need to restart SSH", + ), + ), + ) + @mock.patch(f"{MODPATH}update_ssh_config") + @mock.patch("cloudinit.distros.subp.subp") + def test_restart_ssh_only_when_changes_made_and_ssh_installed( + self, + m_subp, + update_ssh_config, + uses_systemd, + ssh_updated, + cmd, + expected_log, + caplog, + ): + update_ssh_config.return_value = ssh_updated + cloud = get_cloud("ubuntu") + with mock.patch.object( + cloud.distro, "uses_systemd", return_value=uses_systemd + ): + setpass.handle_ssh_pwauth(True, cloud.distro) + if ssh_updated: + m_subp.assert_called_with(cmd, capture=True) + else: + assert [mock.call(cmd, capture=True)] == m_subp.call_args_list + assert expected_log in "\n".join( + r.msg for r in caplog.records if r.levelname == "DEBUG" + ) + + @mock.patch(f"{MODPATH}update_ssh_config", return_value=True) + @mock.patch("cloudinit.distros.subp.subp") + def test_unchanged_value_does_nothing(self, m_subp, update_ssh_config): + """If 'unchanged', then no updates to config and no restart.""" + update_ssh_config.assert_not_called() + cloud = get_cloud("ubuntu") + setpass.handle_ssh_pwauth("unchanged", cloud.distro) + assert [ + mock.call(["systemctl", "status", "ssh"], capture=True) + ] == m_subp.call_args_list + + @pytest.mark.allow_subp_for("systemctl") + @mock.patch("cloudinit.distros.subp.subp") + def test_valid_value_changes_updates_ssh(self, m_subp): + """If value is a valid changed value, then update will be called.""" + cloud = get_cloud("ubuntu") + upname = f"{MODPATH}update_ssh_config" + optname = "PasswordAuthentication" + for n, value in enumerate(util.FALSE_STRINGS + util.TRUE_STRINGS, 1): + optval = "yes" if value in util.TRUE_STRINGS else "no" + with mock.patch(upname, return_value=False) as m_update: + setpass.handle_ssh_pwauth(value, cloud.distro) + assert ( + mock.call({optname: optval}) == m_update.call_args_list[-1] + ) + assert m_subp.call_count == n + + @pytest.mark.parametrize( + [ + "uses_systemd", + "raised_error", + "warning_log", + "debug_logs", + "update_ssh_call_count", + ], + ( + ( + True, + subp.ProcessExecutionError( + stderr="Service is not running.", exit_code=3 + ), + None, + [ + "Writing config 'ssh_pwauth: True'. SSH service" + " 'ssh' will not be restarted because it is stopped.", + "Not restarting SSH service: service is stopped.", + ], + 1, + ), + ( + True, + subp.ProcessExecutionError( + stderr="Service is not installed.", exit_code=4 + ), + "Ignoring config 'ssh_pwauth: True'. SSH service 'ssh' is" + " not installed.", + [], + 0, + ), + ( + True, + subp.ProcessExecutionError( + stderr="Service is not available.", exit_code=2 + ), + "Ignoring config 'ssh_pwauth: True'. SSH service 'ssh'" + " is not available. Error: ", + [], + 0, + ), + ( + False, + subp.ProcessExecutionError( + stderr="Service is not available.", exit_code=25 + ), + None, + [ + "Writing config 'ssh_pwauth: True'. SSH service" + " 'ssh' will not be restarted because it is not running" + " or not available.", + "Not restarting SSH service: service is stopped.", + ], + 1, + ), + ( + False, + subp.ProcessExecutionError( + stderr="Service is not available.", exit_code=3 + ), + None, + [ + "Writing config 'ssh_pwauth: True'. SSH service" + " 'ssh' will not be restarted because it is not running" + " or not available.", + "Not restarting SSH service: service is stopped.", + ], + 1, + ), + ( + False, + subp.ProcessExecutionError( + stderr="Service is not available.", exit_code=4 + ), + None, + [ + "Writing config 'ssh_pwauth: True'. SSH service" + " 'ssh' will not be restarted because it is not running" + " or not available.", + "Not restarting SSH service: service is stopped.", + ], + 1, + ), + ), + ) + @mock.patch(f"{MODPATH}update_ssh_config", return_value=True) + @mock.patch("cloudinit.distros.subp.subp") + def test_no_restart_when_service_is_not_running( + self, + m_subp, + m_update_ssh_config, + uses_systemd, + raised_error, + warning_log, + debug_logs, + update_ssh_call_count, + caplog, + ): + """Write config but don't restart SSH service when not running.""" + cloud = get_cloud("ubuntu") + cloud.distro.manage_service = mock.Mock(side_effect=raised_error) + cloud.distro.uses_systemd = mock.Mock(return_value=uses_systemd) + + setpass.handle_ssh_pwauth(True, cloud.distro) + logs_by_level = {logging.WARNING: [], logging.DEBUG: []} + for _, level, msg in caplog.record_tuples: + logs_by_level[level].append(msg) + if warning_log: + assert warning_log in "\n".join( + logs_by_level[logging.WARNING] + ), logs_by_level + for debug_log in debug_logs: + assert debug_log in logs_by_level[logging.DEBUG] + assert [ + mock.call("status", "ssh") + ] == cloud.distro.manage_service.call_args_list + assert m_update_ssh_config.call_count == update_ssh_call_count + assert m_subp.call_count == 0 + assert cloud.distro.uses_systemd.call_count == 1 + + +def get_chpasswd_calls(cfg, cloud, log): + with mock.patch(f"{MODPATH}subp.subp") as subp: + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle( + "IGNORED", + cfg=cfg, + cloud=cloud, + log=log, + args=[], + ) + assert chpasswd.call_count > 0 + return chpasswd.call_args[0], subp.call_args + + +class TestSetPasswordsHandle: + """Test cc_set_passwords.handle""" + + @mock.patch(f"{MODPATH}subp.subp") + def test_handle_on_empty_config(self, m_subp, caplog): + """handle logs that no password has changed when config is empty.""" + cloud = get_cloud() + setpass.handle("IGNORED", cfg={}, cloud=cloud, log=LOG, args=[]) + assert ( + "Leaving SSH config 'PasswordAuthentication' unchanged. " + "ssh_pwauth=None" + ) in caplog.text + assert [ + mock.call(["systemctl", "status", "ssh"], capture=True) + ] == m_subp.call_args_list + + @mock.patch(f"{MODPATH}subp.subp") + def test_handle_on_chpasswd_list_parses_common_hashes( + self, _m_subp, caplog + ): + """handle parses command password hashes.""" + cloud = get_cloud() + valid_hashed_pwds = [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqYpUW.BrPx/" + "Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52q" + "SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1", + ] + cfg = {"chpasswd": {"list": valid_hashed_pwds}} + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + assert "Handling input for chpasswd as list." in caplog.text + assert "Setting hashed password for ['root', 'ubuntu']" in caplog.text + + first_arg = chpasswd.call_args[0] + for i, val in enumerate(*first_arg): + assert valid_hashed_pwds[i] == ":".join(val) + + @mock.patch(f"{MODPATH}subp.subp") + def test_handle_on_chpasswd_users_parses_common_hashes( + self, _m_subp, caplog + ): + """handle parses command password hashes.""" + cloud = get_cloud() + valid_hashed_pwds = [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqYpUW.BrPx/Dlew1Va", # noqa: E501 + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1", # noqa: E501 + }, + ] + cfg = {"chpasswd": {"users": valid_hashed_pwds}} + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + assert "Handling input for chpasswd as list." not in caplog.text + assert "Setting hashed password for ['root', 'ubuntu']" in caplog.text + first_arg = chpasswd.call_args[0] + for i, (name, password) in enumerate(*first_arg): + assert valid_hashed_pwds[i]["name"] == name + assert valid_hashed_pwds[i]["password"] == password + + @pytest.mark.parametrize( + "user_cfg", + [ + { + "list": [ + "ubuntu:passw0rd", + "sadegh:$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", + ] + }, + { + "users": [ + { + "name": "ubuntu", + "password": "passw0rd", + "type": "text", + }, + { + "name": "sadegh", + "password": "$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", # noqa: E501 + }, + ] + }, + ], + ) + def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( + self, user_cfg, mocker + ): + """BSD don't use chpasswd""" + mocker.patch(f"{MODPATH}util.is_BSD", return_value=True) + m_subp = mocker.patch(f"{MODPATH}subp.subp") + cloud = get_cloud(distro="freebsd") + cfg = {"chpasswd": user_cfg} + with mock.patch.object( + cloud.distro, "uses_systemd", return_value=False + ): + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + assert [ + mock.call( + ["pw", "usermod", "ubuntu", "-h", "0"], + data="passw0rd", + logstring="chpasswd for ubuntu", + ), + mock.call( + ["pw", "usermod", "sadegh", "-H", "0"], + data="$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", + logstring="chpasswd for sadegh", + ), + mock.call(["pw", "usermod", "ubuntu", "-p", "01-Jan-1970"]), + mock.call(["pw", "usermod", "sadegh", "-p", "01-Jan-1970"]), + mock.call(["service", "sshd", "status"], capture=True), + ] == m_subp.call_args_list + + @pytest.mark.parametrize( + "user_cfg", + [ + {"expire": "false", "list": ["root:R", "ubuntu:RANDOM"]}, + { + "expire": "false", + "users": [ + { + "name": "root", + "type": "RANDOM", + }, + { + "name": "ubuntu", + "type": "RANDOM", + }, + ], + }, + ], + ) + def test_random_passwords(self, user_cfg, mocker, caplog): + """handle parses command set random passwords.""" + m_multi_log = mocker.patch(f"{MODPATH}util.multi_log") + mocker.patch(f"{MODPATH}subp.subp") + + cloud = get_cloud() + cfg = {"chpasswd": user_cfg} + + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + dbg_text = "Handling input for chpasswd as list." + if "list" in cfg["chpasswd"]: + assert dbg_text in caplog.text + else: + assert dbg_text not in caplog.text + assert 1 == chpasswd.call_count + user_pass = dict(*chpasswd.call_args[0]) + + assert 1 == m_multi_log.call_count + assert ( + mock.call(mock.ANY, stderr=False, fallback_to_stdout=False) + == m_multi_log.call_args + ) + + assert {"root", "ubuntu"} == set(user_pass.keys()) + written_lines = m_multi_log.call_args[0][0].splitlines() + for password in user_pass.values(): + for line in written_lines: + if password in line: + break + else: + pytest.fail("Password not emitted to console") + + @pytest.mark.parametrize( + "list_def, users_def", + [ + # demonstrate that new addition matches current behavior + ( + { + "chpasswd": { + "list": [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqY" + "pUW.BrPx/Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + "dog:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC" + "7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx" + "3oo1", + "Till:RANDOM", + ] + } + }, + { + "chpasswd": { + "users": [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y" + "5WojbXWqnqYpUW.BrPx/Dlew1Va", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + { + "name": "dog", + "type": "hash", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + { + "name": "Till", + "type": "RANDOM", + }, + ] + } + }, + ), + # Duplicate user: demonstrate no change in current duplicate + # behavior + ( + { + "chpasswd": { + "list": [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqY" + "pUW.BrPx/Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + ] + } + }, + { + "chpasswd": { + "users": [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y" + "5WojbXWqnqYpUW.BrPx/Dlew1Va", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + ] + } + }, + ), + # Duplicate user: demonstrate duplicate across users/list doesn't + # change + ( + { + "chpasswd": { + "list": [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqY" + "pUW.BrPx/Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + ] + } + }, + { + "chpasswd": { + "users": [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y" + "5WojbXWqnqYpUW.BrPx/Dlew1Va", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR5" + "2qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx" + "3oo1", + }, + ], + "list": [ + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + ], + } + }, + ), + ], + ) + def test_chpasswd_parity(self, list_def, users_def): + """Assert that two different configs cause identical calls""" + + cloud = get_cloud() + + def_1 = get_chpasswd_calls(list_def, cloud, LOG) + def_2 = get_chpasswd_calls(users_def, cloud, LOG) + assert def_1 == def_2 + assert def_1[-1] == mock.call( + ["systemctl", "status", "ssh"], capture=True + ) + for val in def_1: + assert val + + +expire_cases = [ + { + "chpasswd": { + "expire": True, + "list": [ + "user1:password", + "user2:R", + "user3:$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", + ], + } + }, + { + "chpasswd": { + "expire": True, + "users": [ + { + "name": "user1", + "password": "password", + "type": "text", + }, + { + "name": "user2", + "type": "RANDOM", + }, + { + "name": "user3", + "password": "$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", # noqa: E501 + }, + ], + } + }, + { + "chpasswd": { + "expire": False, + "list": [ + "user1:password", + "user2:R", + "user3:$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", + ], + } + }, + { + "chpasswd": { + "expire": False, + "users": [ + { + "name": "user1", + "password": "password", + "type": "text", + }, + { + "name": "user2", + "type": "RANDOM", + }, + { + "name": "user3", + "password": "$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", # noqa: E501 + }, + ], + } + }, +] + + +class TestExpire: + @pytest.mark.parametrize("cfg", expire_cases) + def test_expire(self, cfg, mocker, caplog): + cloud = get_cloud() + mocker.patch(f"{MODPATH}subp.subp") + mocker.patch.object(cloud.distro, "chpasswd") + m_expire = mocker.patch.object(cloud.distro, "expire_passwd") + + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + + if bool(cfg["chpasswd"]["expire"]): + assert m_expire.call_args_list == [ + mock.call("user1"), + mock.call("user2"), + mock.call("user3"), + ] + assert ( + "Expired passwords for: ['user1', 'user2', 'user3'] users" + in caplog.text + ) + else: + assert m_expire.call_args_list == [] + assert "Expired passwords" not in caplog.text + + @pytest.mark.parametrize("cfg", expire_cases) + def test_expire_old_behavior(self, cfg, mocker, caplog): + # Previously expire didn't apply to hashed passwords. + # Ensure we can preserve that case on older releases + features.EXPIRE_APPLIES_TO_HASHED_USERS = False + cloud = get_cloud() + mocker.patch(f"{MODPATH}subp.subp") + mocker.patch.object(cloud.distro, "chpasswd") + m_expire = mocker.patch.object(cloud.distro, "expire_passwd") + + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + + if bool(cfg["chpasswd"]["expire"]): + assert m_expire.call_args_list == [ + mock.call("user1"), + mock.call("user2"), + ] + assert ( + "Expired passwords for: ['user1', 'user2'] users" + in caplog.text + ) + else: + assert m_expire.call_args_list == [] + assert "Expired passwords" not in caplog.text + + +class TestSetPasswordsSchema: + @pytest.mark.parametrize( + "config, expectation", + [ + # Test both formats still work + ({"ssh_pwauth": True}, does_not_raise()), + ({"ssh_pwauth": False}, does_not_raise()), + ( + {"ssh_pwauth": "yes"}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: ssh_pwauth: DEPRECATED. Use of" + " non-boolean values for this field is DEPRECATED and" + " will result in an error in a future version of" + " cloud-init." + ), + ), + ), + ( + {"ssh_pwauth": "unchanged"}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: ssh_pwauth: DEPRECATED. Use of" + " non-boolean values for this field is DEPRECATED and" + " will result in an error in a future version of" + " cloud-init." + ), + ), + ), + ( + {"chpasswd": {"list": "blah"}}, + pytest.raises(SchemaValidationError, match="DEPRECATED"), + ), + # Valid combinations + ( + { + "chpasswd": { + "users": [ + { + "name": "what-if-1", + "type": "text", + "password": "correct-horse-battery-staple", + }, + { + "name": "what-if-2", + "type": "hash", + "password": "no-magic-parsing-done-here", + }, + { + "name": "what-if-3", + "password": "type-is-optional-default-" + "value-is-hash", + }, + { + "name": "what-if-4", + "type": "RANDOM", + }, + ] + } + }, + does_not_raise(), + ), + ( + { + "chpasswd": { + "users": [ + { + "name": "what-if-1", + "type": "plaintext", + "password": "type-has-two-legal-values: " + "{'hash', 'text'}", + } + ] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + ( + { + "chpasswd": { + "users": [ + { + "name": "what-if-1", + "type": "RANDOM", + "password": "but you want random?", + } + ] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + ( + {"chpasswd": {"users": [{"password": "."}]}}, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + # when type != RANDOM, password is a required key + ( + { + "chpasswd": { + "users": [{"name": "what-if-1", "type": "hash"}] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + pytest.param( + { + "chpasswd": { + "users": [ + { + "name": "sonata", + "password": "dit", + "dat": "dot", + } + ] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + id="dat_is_an_additional_property", + ), + ( + {"chpasswd": {"users": [{"name": "."}]}}, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + # Test regex + ( + {"chpasswd": {"list": ["user:pass"]}}, + pytest.raises(SchemaValidationError, match="DEPRECATED"), + ), + # Test valid + ({"password": "pass"}, does_not_raise()), + # Test invalid values + ( + {"chpasswd": {"expire": "yes"}}, + pytest.raises( + SchemaValidationError, + match="'yes' is not of type 'boolean'", + ), + ), + ( + {"chpasswd": {"list": ["user"]}}, + pytest.raises(SchemaValidationError), + ), + ( + {"chpasswd": {"list": []}}, + pytest.raises( + SchemaValidationError, match=r"\[\] is too short" + ), + ), + ], + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, expectation): + with expectation: + validate_cloudconfig_schema(config, get_schema(), strict=True) + + +# vi: ts=4 expandtab diff --git a/.pylintrc b/.pylintrc index 3edb0092a..ea6868156 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,8 +25,9 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) # W1514(unspecified-encoding) +# E0012(bad-option-value) -disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401, W1514 +disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401, W1514, E0012 [REPORTS] diff --git a/.travis.yml b/.travis.yml index a529ace13..253295dde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,9 @@ install: # Required so `git describe` will definitely find a tag; see # https://github.com/travis-ci/travis-ci/issues/7422 - git fetch --unshallow + # Not pinning setuptools can cause failures on python 3.7 and 3.8 builds + # See https://github.com/pypa/setuptools/issues/3118 + - pip install setuptools==59.6.0 - pip install tox script: @@ -58,7 +61,7 @@ matrix: sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \; install: - git fetch --unshallow - - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper + - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper wireguard - pip install . - pip install tox # bionic has lxd from deb installed, remove it first to ensure @@ -130,27 +133,20 @@ matrix: TOXENV=lowest-supported PYTEST_ADDOPTS=-v # List all tests run by pytest dist: bionic - - python: 3.6 - env: TOXENV=flake8 - - python: 3.6 - env: TOXENV=mypy - - python: 3.6 - env: TOXENV=pylint - - python: 3.6 - env: TOXENV=black - - python: 3.6 - env: TOXENV=isort - python: 3.7 env: TOXENV=doc install: - git fetch --unshallow + # Not pinning setuptools can cause failures on python 3.7 and 3.8 builds + # See https://github.com/pypa/setuptools/issues/3118 + - pip install setuptools==59.6.0 - sudo apt-get install lintian - pip install tox script: - - make check_spelling - - tox + - make check_spelling && tox # Test all supported Python versions (but at the end, so we schedule # longer-running jobs first) + - python: 3.11-dev - python: "3.10" - python: 3.9 - python: 3.8 diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..6098b6ebc --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "redhat.vscode-yaml" + ] +} diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 819572c66..62628fd50 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -14,7 +14,7 @@ Summary Before any pull request can be accepted, you must do the following: * Sign the Canonical `contributor license agreement`_ -* Add yourself (alphabetically) to the in-repository list that we use +* Add your Github username (alphabetically) to the in-repository list that we use to track CLA signatures: `tools/.github-cla-signers`_ * Add or update any `unit tests`_ accordingly @@ -72,15 +72,15 @@ Follow these steps to submit your first pull request to cloud-init: .. code:: sh - git clone git://github.com/canonical/cloud-init + git clone git@github.com:GH_USER/cloud-init.git cd cloud-init - git remote add GH_USER git@github.com:GH_USER/cloud-init.git - git push GH_USER main + git remote add upstream git@github.com:canonical/cloud-init.git + git push origin main * Read through the cloud-init `Code Review Process`_, so you understand how your changes will end up in cloud-init's codebase. -* Submit your first cloud-init pull request, adding yourself to the +* Submit your first cloud-init pull request, adding your Github username to the in-repository list that we use to track CLA signatures: `tools/.github-cla-signers`_ @@ -144,7 +144,7 @@ Do these things for each feature or bug * Push your changes to your personal GitHub repository:: - git push -u GH_USER my-topic-branch + git push -u origin my-topic-branch * Use your browser to create a pull request: diff --git a/ChangeLog b/ChangeLog index a90a89863..b569e8a23 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,239 @@ +22.3.4 + - Fix Oracle DS primary interface when using IMDS (LP: #1989686) + +22.3.3 + - Fix Oracle DS not setting subnet when using IMDS (LP: #1989686) + +22.3.2 + - azure: define new attribute for pre-22.3 pickles (#1725) + - sources/azure: ensure instance id is always correct (#1727) + +22.3.1 + - Fix v2 interface matching when no MAC (LP: #1986551) + - test: reduce number of network dependencies in flaky test (#1702) + - docs: publish cc_ubuntu_autoinstall docs to rtd (#1696) + - net: Fix EphemeraIPNetwork (#1697) [Alberto Contreras] + - test: make ansible test work across older versions (#1691) + - Networkd multi-address support/fix (#1685) [Teodor Garzdin] + - make: drop broken targets (#1688) + - net: Passthough v2 netconfigs in netplan systems (#1650) + [Alberto Contreras] (LP: #1978543) + - NM ipv6 connection does not work on Azure and Openstack (#1616) + [Emanuele Giuseppe Esposito] + - Fix check_format_tip (#1679) [Alberto Contreras] + - DataSourceVMware: fix var use before init (#1674) + [Andrew Kutz] (LP: #1987005) + - rpm/copr: ensure RPM represents new clean.d dir artifacts (#1680) + - test: avoid centos leaked check of /etc/yum.repos.d/epel-testing.repo + (#1676) + +22.3 + - sources: obj.pkl cache should be written anyime get_data is run (#1669) + - schema: drop release number from version file (#1664) + - pycloudlib: bump to quiet azure HTTP info logs (#1668) + - test: fix wireguard integration tests (#1666) + - Github is deprecating the 18.04 runner starting 12.1 (#1665) + - integration tests: Ensure one setup for all tests (#1661) + - tests: ansible test fixes (#1660) + - Prevent concurrency issue in test_webhook_hander.py (#1658) + - Workaround net_setup_link race with udev (#1655) (LP: #1983516) + - test: drop erroneous lxd assertion, verify command succeeded (#1657) + - Fix Chrony usage on Centos Stream (#1648) [Sven Haardiek] (LP: #1885952) + - sources/azure: handle network unreachable errors for savable PPS (#1642) + [Chris Patterson] + - Return cc_set_hostname to PER_INSTANCE frequency (#1651) (LP: #1983811) + - test: Collect integration test time by default (#1638) + - test: Drop forced package install hack in lxd integration test (#1649) + - schema: Resolve user-data if --system given (#1644) + [Alberto Contreras] (LP: #1983306) + - test: use fake filesystem to avoid file removal (#1647) + [Alberto Contreras] + - tox: Fix tip-flake8 and tip-mypy (#1635) [Alberto Contreras] + - config: Add wireguard config module (#1570) [Fabian Lichtenegger-Lukas] + - tests: can run without azure-cli, tests expect inactive ansible (#1643) + - typing: Type UrlResponse.contents (#1633) [Alberto Contreras] + - testing: fix references to `DEPRECATED.` (#1641) [Alberto Contreras] + - ssh_util: Handle sshd_config.d folder [Alberto Contreras] (LP: #1968873) + - schema: Enable deprecations in cc_update_etc_hosts (#1631) + [Alberto Contreras] + - Add Ansible Config Module (#1579) + - util: Support Idle process state in get_proc_ppid() (#1637) + - schema: Enable deprecations in cc_growpart (#1628) [Alberto Contreras] + - schema: Enable deprecations in cc_users_groups (#1627) + [Alberto Contreras] + - util: Fix error path and parsing in get_proc_ppid() + - main: avoid downloading full contents cmdline urls (#1606) + [Alberto Contreras] (LP: #1937319) + - schema: Enable deprecations in cc_scripts_vendor (#1629) + [Alberto Contreras] + - schema: Enable deprecations in cc_set_passwords (#1630) + [Alberto Contreras] + - sources/azure: add experimental support for preprovisioned os disks + (#1622) [Chris Patterson] + - Remove configobj a_to_u calls (#1632) [Stefano Rivera] + - cc_debug: Drop this module (#1614) [Alberto Contreras] + - schema: add aggregate descriptions in anyOf/oneOf (#1636) + - testing: migrate test_sshutil to pytest (#1617) [Alberto Contreras] + - testing: Fix test_ca_certs integration test (#1626) [Alberto Contreras] + - testing: add support for pycloudlib's pro images (#1604) + [Alberto Contreras] + - testing: migrate test_cc_set_passwords to pytest (#1615) + [Alberto Contreras] + - network: add system_info network activator cloud.cfg overrides (#1619) + (LP: #1958377) + - docs: Align git remotes with uss-tableflip setup (#1624) + [Alberto Contreras] + - testing: cover active config module checks (#1609) [Alberto Contreras] + - lxd: lvm avoid thinpool when kernel module absent + - lxd: enable MTU configuration in cloud-init + - doc: pin doc8 to last passing version + - cc_set_passwords fixes (#1590) + - Modernise importer.py and type ModuleDetails (#1605) [Alberto Contreras] + - config: Def activate_by_schema_keys for t-z (#1613) [Alberto Contreras] + - config: define activate_by_schema_keys for p-r mods (#1611) + [Alberto Contreras] + - clean: add param to remove /etc/machine-id for golden image creation + - config: define `activate_by_schema_keys` for a-f mods (#1608) + [Alberto Contreras] + - config: define activate_by_schema_keys for s mods (#1612) + [Alberto Contreras] + - sources/azure: reorganize tests for network config (#1586) + [Chris Patterson] + - config: Define activate_by_schema_keys for g-n mods (#1610) + [Alberto Contreras] + - meta-schema: add infra to skip inapplicable modules [Alberto Contreras] + - sources/azure: don't set cfg["password"] for default user pw (#1592) + [Chris Patterson] + - schema: activate grub-dpkg deprecations (#1600) [Alberto Contreras] + - docs: clarify user password purposes (#1593) + - cc_lxd: Add btrfs and lvm lxd storage options (SC-1026) (#1585) + - archlinux: Fix distro naming[1] (#1601) [Kristian Klausen] + - cc_ubuntu_autoinstall: support live-installer autoinstall config + - clean: allow third party cleanup scripts in /etc/cloud/clean.d (#1581) + - sources/azure: refactor chassis asset tag handling (#1574) + [Chris Patterson] + - Add "netcho" as contributor (#1591) [Kaloyan Kotlarski] + - testing: drop impish support (#1596) [Alberto Contreras] + - black: fix missed formatting issue which landed in main (#1594) + - bsd: Don't assume that root user is in root group (#1587) + - docs: Fix comment typo regarding use of packages (#1582) + [Peter Mescalchin] + - Update govc command in VMWare walkthrough (#1576) [manioo8] + - Update .github-cla-signers (#1588) [Daniel Mullins] + - Rename the openmandriva user to omv (#1575) [Bernhard Rosenkraenzer] + - sources/azure: increase read-timeout to 60 seconds for wireserver + (#1571) [Chris Patterson] + - Resource leak cleanup (#1556) + - testing: remove appereances of FakeCloud (#1584) [Alberto Contreras] + - Fix expire passwords for hashed passwords (#1577) + [Sadegh Hayeri] (LP: #1979065) + - mounts: fix suggested_swapsize for > 64GB hosts (#1569) [Steven Stallion] + - Update chpasswd schema to deprecate password parsing (#1517) + - tox: Remove entries from default envlist (#1578) (LP: #1980854) + - tests: add test for parsing static dns for existing devices (#1557) + [Jonas Konrad] + - testing: port cc_ubuntu_advantage test to pytest (#1559) + [Alberto Contreras] + - Schema deprecation handling (#1549) [Alberto Contreras] + - Enable pytest to run in parallel (#1568) + - sources/azure: refactor ovf-env.xml parsing (#1550) [Chris Patterson] + - schema: Force stricter validation (#1547) + - ubuntu advantage config: http_proxy, https_proxy (#1512) + [Fabian Lichtenegger-Lukas] + - net: fix interface matching support (#1552) (LP: #1979877) + - Fuzz testing jsonchema (#1499) [Alberto Contreras] + - testing: Wait for changed boot-id in test_status.py (#1548) + - CI: Fix GH pinned-format jobs (#1558) [Alberto Contreras] + - Typo fix (#1560) [Jaime Hablutzel] + - tests: mock dns lookup that causes long timeouts (#1555) + - tox: add unpinned env for do_format and check_format (#1554) + - cc_ssh_import_id: Substitute deprecated warn (#1553) [Alberto Contreras] + - Remove schema errors from log (#1551) (LP: #1978422) (CVE-2022-2084) + - Update WebHookHandler to run as background thread (SC-456) (#1491) + (LP: #1910552) + - testing: Don't run custom cloud dir test on Bionic (#1542) + - bash completion: update schema command (#1543) (LP: #1979547) + - CI: add non-blocking run against the linters tip versions (#1531) + [Paride Legovini] + - Change groups within the users schema to support lists and strings + (#1545) [RedKrieg] + - make it clear which username should go in the contributing doc (#1546) + - Pin setuptools for Travis (SC-1136) (#1540) + - Fix LXD datasource crawl when BOOT enabled (#1537) + - testing: Fix wrong path in dual stack test (#1538) + - cloud-config: honor cloud_dir setting (#1523) + [Alberto Contreras] (LP: #1976564) + - Add python3-debconf to pkg-deps.json Build-Depends (#1535) + [Alberto Contreras] + - redhat spec: udev/rules.d lives under /usr/lib on rhel-based systems + (#1536) + - tests/azure: add test coverage for DisableSshPasswordAuthentication + (#1534) [Chris Patterson] + - summary: Add david-caro to the cla signers (#1527) [David Caro] + - Add support for OpenMandriva (https://openmandriva.org/) (#1520) + [Bernhard Rosenkraenzer] + - tests/azure: refactor ovf creation (#1533) [Chris Patterson] + - Improve DataSourceOVF error reporting when script disabled (#1525) [rong] + - tox: integration-tests-jenkins: softfail if only some test failed + (#1528) [Paride Legovini] + - CI: drop linters from Travis CI (moved to GH Actions) (#1530) + [Paride Legovini] + - sources/azure: remove unused encoding support for customdata (#1526) + [Chris Patterson] + - sources/azure: remove unused metadata captured when parsing ovf (#1524) + [Chris Patterson] + - sources/azure: remove dscfg parsing from ovf-env.xml (#1522) + [Chris Patterson] + - Remove extra space from ec2 dual stack crawl message (#1521) + - tests/azure: use namespaces in generated ovf-env.xml documents (#1519) + [Chris Patterson] + - setup.py: adjust udev/rules default path (#1513) + [Emanuele Giuseppe Esposito] + - Add python3-deconf dependency (#1506) [Alberto Contreras] + - Change match macadress param for network v2 config (#1518) + [Henrique Caricatti Capozzi] + - sources/azure: remove unused userdata property from ovf (#1516) + [Chris Patterson] + - sources/azure: minor refactoring to network config generation (#1497) + [Chris Patterson] + - net: Implement link-local ephemeral ipv6 + - Rename function to avoid confusion (#1501) + - Fix cc_phone_home requiring 'tries' (#1500) (LP: #1977952) + - datasources: replace networking functions with stdlib and cloudinit.net + code + - Remove xenial references (#1472) [Alberto Contreras] + - Oracle ds changes (#1474) [Alberto Contreras] (LP: #1967942) + - improve runcmd docs (#1498) + - add 3.11-dev to Travis CI (#1493) + - Only run github actions on pull request (#1496) + - Fix integration test client creation (#1494) [Alberto Contreras] + - tox: add link checker environment, fix links (#1480) + - cc_ubuntu_advantage: Fix doc (#1487) [Alberto Contreras] + - cc_yum_add_repo: Fix repo id canonicalization (#1489) + [Alberto Contreras] (LP: #1975818) + - Add linitio as contributor in the project (#1488) [Kevin Allioli] + - net-convert: use yaml.dump for debugging python NetworkState obj (#1484) + (LP: #1975907) + - test_schema: no relative $ref URLs, replace $ref with local path (#1486) + - cc_set_hostname: do not write "localhost" when no hostname is given + (#1453) [Emanuele Giuseppe Esposito] + - Update .github-cla-signers (#1478) [rong] + - schema: write_files defaults, versions $ref full URL and add vscode + (#1479) + - docs: fix external links, add one more to the list (#1477) + - doc: Document how to change module frequency (#1481) + - tests: bump pycloudlib (#1482) + - tests: bump pycloudlib pinned commit for kinetic Azure (#1476) + - testing: fix test_status.py (#1475) + - integration tests: If KEEP_INSTANCE = True, log IP (#1473) + - Drop mypy excluded files (#1454) [Alberto Contreras] + - Docs additions (#1470) + - Add "formatting tests" to Github Actions + - Remove unused arguments in function signature (#1471) + - Changelog: correct errant classification of LP issues as GH (#1464) + - Use Network-Manager and Netplan as default renderers for RHEL and Fedora + (#1465) [Emanuele Giuseppe Esposito] + 22.2 - Fix test due to caplog incompatibility (#1461) [Alberto Contreras] - Align rhel custom files with upstream (#1431) @@ -21,16 +257,15 @@ - tests: cc_set_passoword update for systemd, non-systemd distros (#1449) - Fix bug in url_helper/dual_stack() logging (#1426) - schema: render schema paths from _CustomSafeLoaderWithMarks (#1391) - (GH: SC-929) - testing: Make integration tests kinetic friendly (#1441) - Handle error if SSH service no present. (#1422) - [Alberto Contreras] (GH: #1969526) + [Alberto Contreras] (LP: #1969526) - Fix network-manager activator availability and order (#1438) - sources/azure: remove reprovisioning marker (#1414) [Chris Patterson] - upstart: drop vestigial support for upstart (#1421) - testing: Ensure NoCloud detected in test (#1439) - Update .github-cla-signers kallioli [Kevin Allioli] - - Consistently strip top-level network key (#1417) (GH: #1906187) + - Consistently strip top-level network key (#1417) (LP: #1906187) - testing: Fix LXD VM metadata test (#1430) - testing: Add NoCloud setup for NoCloud test (#1425) - Update linters and adapt code for compatibility (#1434) [Paride Legovini] @@ -43,9 +278,9 @@ - tests: verify_ordered_items fallback to re.escape if needed (#1420) - Misc module cleanup (#1418) - docs: Fix doc warnings and enable errors (#1419) - [Alberto Contreras] (GH: #1876341) + [Alberto Contreras] (LP: #1876341) - Refactor cloudinit.sources.NetworkConfigSource to enum (#1413) - [Alberto Contreras] (GH: #1874875) + [Alberto Contreras] (LP: #1874875) - Don't fail if IB and Ethernet devices 'collide' (#1411) - Use cc_* module meta defintion over hardcoded vars (SC-888) (#1385) - Fix cc_rsyslog.py initialization (#1404) [Alberto Contreras] @@ -57,13 +292,13 @@ - Allow growpart to resize encrypted partitions (#1316) - Fix typo in integration_test.rst (#1405) [Alberto Contreras] - cloudinit.net refactor: apply_network_config_names (#1388) - [Alberto Contreras] (GH: #1884602) + [Alberto Contreras] (LP: #1884602) - tests/azure: add fixtures for hardcoded paths (markers and data_dir) (#1399) [Chris Patterson] - testing: Add responses workaround for focal/impish (#1403) - cc_ssh_import_id: fix is_key_in_nested_dict to avoid early False - Fix ds-identify not detecting NoCloud seed in config (#1381) - (GH: #1876375) + (LP: #1876375) - sources/azure: retry dhcp for failed processes (#1401) [Chris Patterson] - Move notes about refactorization out of CONTRIBUTING.rst (#1389) - Shave ~8ms off generator runtime (#1387) @@ -78,28 +313,27 @@ - sources/azure: only wait for primary nic to be attached during restore (#1378) [Anh Vo] - cc_ntp: migrated legacy schema to cloud-init-schema.json (#1384) - (GH: SC-803) - Network functions refactor and bugfixes (#1383) - schema: add JSON defs for modules cc_users_groups (#1379) - (GH: SC-928, SC-846, SC-897, #1858930) + (LP: #1858930) - Fix doc typo (#1382) [Alberto Contreras] - Add support for dual stack IPv6/IPv4 IMDS to Ec2 (#1160) - - Fix KeyError when rendering sysconfig IPv6 routes (#1380) (GH: #1958506) + - Fix KeyError when rendering sysconfig IPv6 routes (#1380) (LP: #1958506) - Return a namedtuple from subp() (#1376) - Mypy stubs and other tox maintenance (SC-920) (#1374) - Distro Compatibility Fixes (#1375) - Pull in Gentoo patches (#1372) - schema: add json defs for modules U-Z (#1360) - (GH: #1858928, #1858929, #1858931, #1858932) + (LP: #1858928, #1858929, #1858931, #1858932) - util: atomically update sym links to avoid Suppress FileNotFoundError - when reading status (#1298) [Adam Collard] (GH: LP:1962150) + when reading status (#1298) [Adam Collard] (LP: #1962150) - schema: add json defs for modules scripts-timezone (SC-801) (#1365) - docs: Add first tutorial (SC-900) (#1368) - BUG 1473527: module ssh-authkey-fingerprints fails Input/output error… - (#1340) [Andrew Lee] (GH: #1473527) + (#1340) [Andrew Lee] (LP: #1473527) - add arch hosts template (#1371) - ds-identify: detect LXD for VMs launched from host with > 5.10 kernel - (#1370) (GH: #1968085) + (#1370) (LP: #1968085) - Support EC2 tags in instance metadata (#1309) [Eduardo Dobay] - schema: add json defs for modules e-install (SC-651) (#1366) - Improve "(no_create_home|system): true" test (#1367) [Jeffrey 'jf' Lim] @@ -122,7 +356,7 @@ - testing: Add missing is_FreeBSD mock to networking test (#1353) - Add --no-update to add-apt-repostory call (SC-880) (#1337) - schema: add json defs for modules K-L (#1321) - (GH: #1858899, #1858900, #1858901, #1858902) + (LP: #1858899, #1858900, #1858901, #1858902) - docs: Re-order readthedocs install (#1354) - Stop cc_ssh_authkey_fingerprints from ALWAYS creating home (#1343) [Jeffrey 'jf' Lim] @@ -131,14 +365,14 @@ - sources/azure: move get_ip_from_lease_value out of shim (#1324) [Chris Patterson] - Fix cloud-init status --wait when no datasource found (#1349) - (GH: #1966085) + (LP: #1966085) - schema: add JSON defs for modules resize-salt (SC-654) (#1341) - Add myself as a future contributor (#1345) [Neal Gompa (ニール・ゴンパ)] - Update .github-cla-signers (#1342) [Jeffrey 'jf' Lim] - add Requires=cloud-init-hotplugd.socket in cloud-init-hotplugd.service file (#1335) [yangzz-97] - Fix sysconfig render when set-name is missing (#1327) - [Andrew Kutz] (GH: #1855945) + [Andrew Kutz] (LP: #1855945) - Refactoring helper funcs out of NetworkState (#1336) [Andrew Kutz] - url_helper: add tuple support for readurl timeout (#1328) [Chris Patterson] @@ -162,7 +396,7 @@ - Doc cleanups (#1317) - docs improvements (#1312) - add support for jinja do statements, add unit test (#1314) - [Paul Bruno] (GH: #1962759) + [Paul Bruno] (LP: #1962759) - sources/azure: prevent tight loops for DHCP retries (#1285) [Chris Patterson] - net/dhcp: surface type of DHCP lease failure to caller (#1276) @@ -177,7 +411,7 @@ [Adam Collard] - check for existing symlink while force creating symlink (#1281) [Shreenidhi Shedi] - - Do not silently ignore integer uid (#1280) (GH: #1875772) + - Do not silently ignore integer uid (#1280) (LP: #1875772) - tests: create a IPv4/IPv6 VPC in Ec2 integration tests (#1291) - Integration test fix ppa (#1296) - tests: on official EC2. cloud-id actually startswith aws not ec2 (#1289) @@ -319,8 +553,7 @@ - sources/azure: remove unnecessary hostname bounce (#1143) [Chris Patterson] - find_devs/openbsd: accept ISO on disk (#1132) - [Gonéri Le Bouder] (GH: - https://github.com/ContainerCraft/kmi/issues/12) + [Gonéri Le Bouder] - Improve error log message when mount failed (#1140) [Ksenija Stanojevic] - add KsenijaS as a contributor (#1145) [Ksenija Stanojevic] - travis - don't run integration tests if no deb (#1139) @@ -328,14 +561,14 @@ - testing: Add deterministic test id (#1138) - mock sleep() in azure test (#1137) - Add miraclelinux support (#1128) [Haruki TSURUMOTO] - - docs: Make MACs lowercase in network config (#1135) (GH: #1876941) + - docs: Make MACs lowercase in network config (#1135) (LP: #1876941) - Add Strict Metaschema Validation (#1101) - update dead link (#1133) - cloudinit/net: handle two different routes for the same ip (#1124) [Emanuele Giuseppe Esposito] - docs: pin mistune dependency (#1134) - Reorganize unit test locations under tests/unittests (#1126) - - Fix exception when no activator found (#1129) (GH: #1948681) + - Fix exception when no activator found (#1129) (LP: #1948681) - jinja: provide and document jinja-safe key aliases in instance-data (SC-622) (#1123) - testing: Remove date from final_message test (SC-638) (#1127) @@ -352,7 +585,7 @@ - lxd: add preference for LXD cloud-init.* config keys over user keys (#1108) - VMware: source /etc/network/interfaces.d/* on Debian - [chengcheng-chcheng] (GH: #1950136) + [chengcheng-chcheng] (LP: #1950136) - Add cjp256 as contributor (#1109) [Chris Patterson] - integration_tests: Ensure log directory exists before symlinking to it (#1110) @@ -362,8 +595,8 @@ - tests: specialize lxd_discovery test for lxd_vm vendordata (#1106) - Add convenience symlink to integration test output (#1105) - Fix for set-name bug in networkd renderer (#1100) - [Andrew Kutz] (GH: #1949407) - - Wait for apt lock (#1034) (GH: #1944611) + [Andrew Kutz] (LP: #1949407) + - Wait for apt lock (#1034) (LP: #1944611) - testing: stop chef test from running on openstack (#1102) - alpine.py: add options to the apk upgrade command (#1089) [dermotbradley] diff --git a/Makefile b/Makefile index 9584ccc1c..2acf132ed 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f ) YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) PYTHON = python3 -PIP_INSTALL := pip3 install NUM_ITER ?= 100 @@ -56,14 +55,6 @@ ci-deps-ubuntu: ci-deps-centos: @$(PYTHON) $(CWD)/tools/read-dependencies --distro centos --test-distro -pip-requirements: - @echo "Installing cloud-init dependencies..." - $(PIP_INSTALL) -r "$@.txt" -q - -pip-test-requirements: - @echo "Installing cloud-init test dependencies..." - $(PIP_INSTALL) -r "$@.txt" -q - test: unittest check_version: @@ -128,13 +119,20 @@ deb-src: doc: tox -e doc +fmt: + tox -e do_format && tox -e check_format + +fmt-tip: + tox -e do_format_tip && tox -e check_format_tip + # Spell check && filter false positives _CHECK_SPELLING := find doc -type f -exec spellintian {} + | \ grep -v -e 'doc/rtd/topics/cli.rst: modules modules' \ -e 'doc/examples/cloud-config-mcollective.txt: WARNING WARNING' \ -e 'doc/examples/cloud-config-power-state.txt: Bye Bye' \ -e 'doc/examples/cloud-config.txt: Bye Bye' \ - -e 'doc/rtd/topics/cli.rst: DOCS DOCS' + -e 'doc/rtd/topics/cli.rst: DOCS DOCS' \ + -e 'dependant' # For CI we require a failing return code when spellintian finds spelling errors @@ -167,6 +165,6 @@ fix_spelling: sh .PHONY: all check test flake8 clean rpm srpm deb deb-src yaml -.PHONY: check_version pip-test-requirements pip-requirements clean_pyc +.PHONY: check_version clean_pyc .PHONY: unittest style-check fix_spelling render-template benchmark-generator .PHONY: clean_pytest clean_packaging check_spelling clean_release doc diff --git a/README.md b/README.md index f2a745f87..64a1635d5 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Alpine Linux
ArchLinux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
openEuler
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux/CloudLinux/MIRACLE LINUX
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| +| Alpine Linux
Arch Linux
Debian
DragonFlyBSD
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
openEuler
OpenMandriva
RHEL/CentOS/AlmaLinux/Rocky/PhotonOS/Virtuozzo/EuroLinux/CloudLinux/MIRACLE LINUX
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
DigitalOcean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Vultr
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)
VMware















| ## To start developing cloud-init diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init index 1eceb472b..579005d29 100644 --- a/bash_completion/cloud-init +++ b/bash_completion/cloud-init @@ -10,7 +10,7 @@ _cloudinit_complete() cur_word="${COMP_WORDS[COMP_CWORD]}" prev_word="${COMP_WORDS[COMP_CWORD-1]}" - subcmds="analyze clean collect-logs devel dhclient-hook features init modules query single status" + subcmds="analyze clean collect-logs devel dhclient-hook features init modules query schema single status" base_params="--help --file --version --debug --force" case ${COMP_CWORD} in 1) @@ -28,7 +28,7 @@ _cloudinit_complete() COMPREPLY=($(compgen -W "--help --tarfile --include-userdata" -- $cur_word)) ;; devel) - COMPREPLY=($(compgen -W "--help hotplug-hook schema net-convert" -- $cur_word)) + COMPREPLY=($(compgen -W "--help hotplug-hook net-convert" -- $cur_word)) ;; dhclient-hook) COMPREPLY=($(compgen -W "--help up down" -- $cur_word)) diff --git a/cloudinit/analyze/__main__.py b/cloudinit/analyze/__main__.py index 36a5be786..df08d46c9 100644 --- a/cloudinit/analyze/__main__.py +++ b/cloudinit/analyze/__main__.py @@ -6,6 +6,7 @@ import re import sys from datetime import datetime +from typing import IO from cloudinit.util import json_dumps @@ -192,6 +193,7 @@ def analyze_boot(name, args): } outfh.write(status_map[status_code].format(**kwargs)) + clean_io(infh, outfh) return status_code @@ -218,6 +220,7 @@ def analyze_blame(name, args): outfh.write("\n".join(srecs) + "\n") outfh.write("\n") outfh.write("%d boot records analyzed\n" % (idx + 1)) + clean_io(infh, outfh) def analyze_show(name, args): @@ -254,12 +257,14 @@ def analyze_show(name, args): ) outfh.write("\n".join(record) + "\n") outfh.write("%d boot records analyzed\n" % (idx + 1)) + clean_io(infh, outfh) def analyze_dump(name, args): """Dump cloud-init events in json format""" (infh, outfh) = configure_io(args) outfh.write(json_dumps(_get_events(infh)) + "\n") + clean_io(infh, outfh) def _get_events(infile): @@ -293,6 +298,14 @@ def configure_io(args): return (infh, outfh) +def clean_io(*file_handles: IO) -> None: + """close filehandles""" + for file_handle in file_handles: + if file_handle in (sys.stdin, sys.stdout): + continue + file_handle.close() + + if __name__ == "__main__": parser = get_parser() args = parser.parse_args() diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index abfa09131..04621f124 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -8,7 +8,6 @@ import datetime import json import os -import sys import time from cloudinit import subp, util @@ -257,25 +256,21 @@ def gather_timestamps_using_systemd(): status = SUCCESS_CODE # lxc based containers do not set their monotonic zero point to be when # the container starts, instead keep using host boot as zero point - # time.CLOCK_MONOTONIC_RAW is only available in python 3.3 if util.is_container(): # clock.monotonic also uses host boot as zero point - if sys.version_info >= (3, 3): - base_time = float(time.time()) - float(time.monotonic()) - # TODO: lxcfs automatically truncates /proc/uptime to seconds - # in containers when https://github.com/lxc/lxcfs/issues/292 - # is fixed, util.uptime() should be used instead of stat on - try: - file_stat = os.stat("/proc/1/cmdline") - kernel_start = file_stat.st_atime - except OSError as err: - raise RuntimeError( - "Could not determine container boot " - "time from /proc/1/cmdline. ({})".format(err) - ) from err - status = CONTAINER_CODE - else: - status = FAIL_CODE + base_time = float(time.time()) - float(time.monotonic()) + # TODO: lxcfs automatically truncates /proc/uptime to seconds + # in containers when https://github.com/lxc/lxcfs/issues/292 + # is fixed, util.uptime() should be used instead of stat on + try: + file_stat = os.stat("/proc/1/cmdline") + kernel_start = file_stat.st_atime + except OSError as err: + raise RuntimeError( + "Could not determine container boot " + "time from /proc/1/cmdline. ({})".format(err) + ) from err + status = CONTAINER_CODE kernel_end = base_time + delta_k_end cloudinit_sysd = base_time + delta_ci_s diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 92068aa9b..aa3a6c5c3 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -3,6 +3,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """Cloud-init apport interface""" +from cloudinit.cmd.devel import read_cfg_paths try: from apport.hookutils import ( @@ -53,7 +54,11 @@ # Potentially clear text collected logs CLOUDINIT_LOG = "/var/log/cloud-init.log" CLOUDINIT_OUTPUT_LOG = "/var/log/cloud-init-output.log" -USER_DATA_FILE = "/var/lib/cloud/instance/user-data.txt" # Optional + + +def _get_user_data_file() -> str: + paths = read_cfg_paths() + return paths.get_ipath_cur("userdata_raw") def attach_cloud_init_logs(report, ui=None): @@ -106,18 +111,19 @@ def attach_cloud_info(report, ui=None): def attach_user_data(report, ui=None): """Optionally provide user-data if desired.""" if ui: + user_data_file = _get_user_data_file() prompt = ( "Your user-data or cloud-config file can optionally be provided" " from {0} and could be useful to developers when addressing this" " bug. Do you wish to attach user-data to this bug?".format( - USER_DATA_FILE + user_data_file ) ) response = ui.yesno(prompt) if response is None: raise StopIteration # User cancelled if response: - attach_file(report, USER_DATA_FILE, "user_data.txt") + attach_file(report, user_data_file, "user_data.txt") def add_bug_tags(report): diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py index 1a0176082..65d3eecec 100755 --- a/cloudinit/cmd/clean.py +++ b/cloudinit/cmd/clean.py @@ -11,8 +11,9 @@ import os import sys +from cloudinit import settings from cloudinit.stages import Init -from cloudinit.subp import ProcessExecutionError, subp +from cloudinit.subp import ProcessExecutionError, runparts, subp from cloudinit.util import ( del_dir, del_file, @@ -21,6 +22,8 @@ is_link, ) +ETC_MACHINE_ID = "/etc/machine-id" + def get_parser(parser=None): """Build or extend an arg parser for clean utility. @@ -47,6 +50,15 @@ def get_parser(parser=None): dest="remove_logs", help="Remove cloud-init logs.", ) + parser.add_argument( + "--machine-id", + action="store_true", + default=False, + help=( + "Remove /etc/machine-id for golden image creation." + " Next boot generates a new machine-id." + ), + ) parser.add_argument( "-r", "--reboot", @@ -94,12 +106,21 @@ def remove_artifacts(remove_logs, remove_seed=False): except OSError as e: error("Could not remove {0}: {1}".format(path, str(e))) return 1 + try: + runparts(settings.CLEAN_RUNPARTS_DIR) + except Exception as e: + error( + f"Failure during run-parts of {settings.CLEAN_RUNPARTS_DIR}: {e}" + ) + return 1 return 0 def handle_clean_args(name, args): """Handle calls to 'cloud-init clean' as a subcommand.""" exit_code = remove_artifacts(args.remove_logs, args.remove_seed) + if args.machine_id: + del_file(ETC_MACHINE_ID) if exit_code == 0 and args.reboot: cmd = ["shutdown", "-r", "now"] try: diff --git a/cloudinit/cmd/cloud_id.py b/cloudinit/cmd/cloud_id.py index 34160f8c0..567d341a5 100755 --- a/cloudinit/cmd/cloud_id.py +++ b/cloudinit/cmd/cloud_id.py @@ -76,7 +76,8 @@ def handle_args(name, args): return 3 try: - instance_data = json.load(open(args.instance_data)) + with open(args.instance_data) as file: + instance_data = json.load(file) except IOError: return error( "File not found '%s'. Provide a path to instance data json file" diff --git a/cloudinit/cmd/devel/hotplug_hook.py b/cloudinit/cmd/devel/hotplug_hook.py index 29439911e..f95e8cc04 100755 --- a/cloudinit/cmd/devel/hotplug_hook.py +++ b/cloudinit/cmd/devel/hotplug_hook.py @@ -10,7 +10,7 @@ from cloudinit import log, reporting, stages from cloudinit.event import EventScope, EventType -from cloudinit.net import activators, read_sys_net_safe +from cloudinit.net import read_sys_net_safe from cloudinit.net.network_state import parse_net_config_data from cloudinit.reporting import events from cloudinit.sources import DataSource, DataSourceNotFoundException @@ -132,7 +132,7 @@ def apply(self): bring_up=False, ) interface_name = os.path.basename(self.devpath) - activator = activators.select_activator() + activator = self.datasource.distro.network_activator() if self.action == "add": if not activator.bring_up_interface(interface_name): raise RuntimeError( @@ -202,12 +202,12 @@ def handle_hotplug(hotplug_init: Init, devpath, subsystem, udevaction): return handler_cls = SUBSYSTEM_PROPERTES_MAP[subsystem][0] LOG.debug("Creating %s event handler", subsystem) - event_handler = handler_cls( + event_handler: UeventHandler = handler_cls( datasource=datasource, devpath=devpath, action=udevaction, success_fn=hotplug_init._write_to_cache, - ) # type: UeventHandler + ) wait_times = [1, 3, 5, 10, 30] last_exception = Exception("Bug while processing hotplug event.") for attempt, wait in enumerate(wait_times): diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index fbe8c500f..a87b7043f 100755 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -12,6 +12,7 @@ import sys from datetime import datetime +from cloudinit.cmd.devel import read_cfg_paths from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.subp import ProcessExecutionError, subp from cloudinit.temp_utils import tempdir @@ -19,7 +20,11 @@ CLOUDINIT_LOGS = ["/var/log/cloud-init.log", "/var/log/cloud-init-output.log"] CLOUDINIT_RUN_DIR = "/run/cloud-init" -USER_DATA_FILE = "/var/lib/cloud/instance/user-data.txt" # Optional + + +def _get_user_data_file() -> str: + paths = read_cfg_paths() + return paths.get_ipath_cur("userdata_raw") def get_parser(parser=None): @@ -53,6 +58,7 @@ def get_parser(parser=None): " Default: cloud-init.tar.gz" ), ) + user_data_file = _get_user_data_file() parser.add_argument( "--include-userdata", "-u", @@ -61,7 +67,7 @@ def get_parser(parser=None): dest="userdata", help=( "Optionally include user-data from {0} which could contain" - " sensitive information.".format(USER_DATA_FILE) + " sensitive information.".format(user_data_file) ), ) return parser @@ -104,7 +110,7 @@ def _collect_file(path, out_dir, verbosity): _debug("file %s did not exist\n" % path, 2, verbosity) -def collect_logs(tarfile, include_userdata, verbosity=0): +def collect_logs(tarfile, include_userdata: bool, verbosity=0): """Collect all cloud-init logs and tar them up into the provided tarfile. @param tarfile: The path of the tar-gzipped file to create. @@ -152,7 +158,8 @@ def collect_logs(tarfile, include_userdata, verbosity=0): for log in CLOUDINIT_LOGS: _collect_file(log, log_dir, verbosity) if include_userdata: - _collect_file(USER_DATA_FILE, log_dir, verbosity) + user_data_file = _get_user_data_file() + _collect_file(user_data_file, log_dir, verbosity) run_dir = os.path.join(log_dir, "run") ensure_dir(run_dir) if os.path.exists(CLOUDINIT_RUN_DIR): diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py index e3f58e908..269d72cd5 100755 --- a/cloudinit/cmd/devel/net_convert.py +++ b/cloudinit/cmd/devel/net_convert.py @@ -7,6 +7,8 @@ import os import sys +import yaml + from cloudinit import distros, log, safeyaml from cloudinit.net import ( eni, @@ -124,26 +126,21 @@ def handle_args(name, args): json.loads(net_data), known_macs=known_macs ) elif args.kind == "azure-imds": - pre_ns = azure.parse_network_config(json.loads(net_data)) + pre_ns = azure.generate_network_config_from_instance_network_metadata( + json.loads(net_data)["network"] + ) elif args.kind == "vmware-imc": config = ovf.Config(ovf.ConfigFile(args.network_data.name)) pre_ns = ovf.get_network_config_from_conf(config, False) - ns = network_state.parse_net_config_data(pre_ns) - - if args.debug: - sys.stderr.write( - "\n".join(["", "Internal State", safeyaml.dumps(ns), ""]) - ) distro_cls = distros.fetch(args.distro) distro = distro_cls(args.distro, {}, None) - config = {} if args.output_kind == "eni": r_cls = eni.Renderer config = distro.renderer_configs.get("eni") elif args.output_kind == "netplan": r_cls = netplan.Renderer - config = distro.renderer_configs.get("netplan") + config = distro.renderer_configs.get("netplan", {}) # don't run netplan generate/apply config["postcmds"] = False # trim leading slash @@ -163,6 +160,11 @@ def handle_args(name, args): raise RuntimeError("Invalid output_kind") r = r_cls(config=config) + ns = network_state.parse_net_config_data(pre_ns, renderer=r) + + if args.debug: + sys.stderr.write("\n".join(["", "Internal State", yaml.dump(ns), ""])) + sys.stderr.write( "".join( [ diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 54b76e9c8..6134d7c47 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -21,6 +21,7 @@ import sys import time import traceback +from typing import Tuple from cloudinit import patcher from cloudinit.config.modules import Modules @@ -47,6 +48,7 @@ from cloudinit.config import cc_set_hostname from cloudinit import dhclient_hook +from cloudinit.cmd.devel import read_cfg_paths # Welcome message template @@ -142,7 +144,7 @@ def parse_cmdline_url(cmdline, names=("cloud-config-url", "url")): raise KeyError("No keys (%s) found in string '%s'" % (cmdline, names)) -def attempt_cmdline_url(path, network=True, cmdline=None): +def attempt_cmdline_url(path, network=True, cmdline=None) -> Tuple[int, str]: """Write data from url referenced in command line to path. path: a file to write content to if downloaded. @@ -189,7 +191,7 @@ def attempt_cmdline_url(path, network=True, cmdline=None): return (level, m) - kwargs = {"url": url, "timeout": 10, "retries": 2} + kwargs = {"url": url, "timeout": 10, "retries": 2, "stream": True} if network or path_is_local: level = logging.WARN kwargs["sec_between"] = 1 @@ -201,22 +203,43 @@ def attempt_cmdline_url(path, network=True, cmdline=None): header = b"#cloud-config" try: resp = url_helper.read_file_or_url(**kwargs) + sniffed_content = b"" if resp.ok(): - data = resp.contents - if not resp.contents.startswith(header): + is_cloud_cfg = True + if isinstance(resp, url_helper.UrlResponse): + try: + sniffed_content += next( + resp.iter_content(chunk_size=len(header)) + ) + except StopIteration: + pass + if not sniffed_content.startswith(header): + is_cloud_cfg = False + elif not resp.contents.startswith(header): + is_cloud_cfg = False + if is_cloud_cfg: + if cmdline_name == "url": + LOG.warning( + "DEPRECATED: `url` kernel command line key is" + " deprecated for providing cloud-config via URL." + " Please use `cloud-config-url` kernel command line" + " parameter instead" + ) + else: if cmdline_name == "cloud-config-url": level = logging.WARN else: level = logging.INFO return ( level, - "contents of '%s' did not start with %s" % (url, header), + f"contents of '{url}' did not start with {str(header)}", ) else: return ( level, "url '%s' returned code %s. Ignoring." % (url, resp.code), ) + data = sniffed_content + resp.contents except url_helper.UrlError as e: return (level, "retrieving url '%s' failed: %s" % (url, e)) @@ -455,7 +478,10 @@ def main_init(name, args): # Validate user-data adheres to schema definition if os.path.exists(init.paths.get_ipath_cur("userdata_raw")): validate_cloudconfig_schema( - config=init.cfg, strict=False, log_details=False + config=init.cfg, + strict=False, + log_details=False, + log_deprecations=True, ) else: LOG.debug("Skipping user-data validation. No user-data found.") @@ -663,7 +689,8 @@ def main_single(name, args): def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: - data_d = os.path.normpath("/var/lib/cloud/data") + paths = read_cfg_paths() + data_d = paths.get_cpath("data") if link_d is None: link_d = os.path.normpath("/run/cloud-init") @@ -782,7 +809,7 @@ def _maybe_persist_instance_data(init): init.paths.run_dir, sources.INSTANCE_JSON_FILE ) if not os.path.exists(instance_data_file): - init.datasource.persist_instance_data() + init.datasource.persist_instance_data(write_cache=False) def _maybe_set_hostname(init, stage, retry_stage): @@ -792,7 +819,7 @@ def _maybe_set_hostname(init, stage, retry_stage): @param retry_stage: String represented logs upon error setting hostname. """ cloud = init.cloudify() - (hostname, _fqdn) = util.get_hostname_fqdn( + (hostname, _fqdn, _) = util.get_hostname_fqdn( init.cfg, cloud, metadata_only=True ) if hostname: # meta-data or user-data hostname content diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index b9347200a..2dcd8e44a 100755 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -150,7 +150,6 @@ def _read_instance_data(instance_data, user_data, vendor_data) -> dict: :raise: IOError/OSError on absence of instance-data.json file or invalid access perms. """ - paths = None uid = os.getuid() if not all([instance_data, user_data, vendor_data]): paths = read_cfg_paths() diff --git a/cloudinit/config/cc_ansible.py b/cloudinit/config/cc_ansible.py new file mode 100644 index 000000000..923092720 --- /dev/null +++ b/cloudinit/config/cc_ansible.py @@ -0,0 +1,188 @@ +"""ansible enables running on first boot either ansible-pull""" +import abc +import logging +import os +import re +import sys +from copy import deepcopy +from textwrap import dedent +from typing import Optional + +from cloudinit.cloud import Cloud +from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.distros import ALL_DISTROS +from cloudinit.settings import PER_INSTANCE +from cloudinit.subp import subp, which +from cloudinit.util import Version, get_cfg_by_path + +meta: MetaSchema = { + "id": "cc_ansible", + "name": "Ansible", + "title": "Configure ansible for instance", + "frequency": PER_INSTANCE, + "distros": [ALL_DISTROS], + "activate_by_schema_keys": ["ansible"], + "description": dedent( + """\ + This module provides ``ansible`` integration for + augmenting cloud-init's configuration of the local + node. + + + This module installs ansible during boot and + then uses ``ansible-pull`` to run the playbook + repository at the remote URL. + """ + ), + "examples": [ + dedent( + """\ + #cloud-config + ansible: + install-method: distro + pull: + url: "https://github.com/holmanb/vmboot.git" + playbook-name: ubuntu.yml + """ + ), + dedent( + """\ + #cloud-config + ansible: + package-name: ansible-core + install-method: pip + pull: + url: "https://github.com/holmanb/vmboot.git" + playbook-name: ubuntu.yml + """ + ), + ], +} + +__doc__ = get_meta_doc(meta) +LOG = logging.getLogger(__name__) + + +class AnsiblePull(abc.ABC): + cmd_version: list = [] + cmd_pull: list = [] + env: dict = os.environ.copy() + + def get_version(self) -> Optional[Version]: + stdout, _ = subp(self.cmd_version, env=self.env) + first_line = stdout.splitlines().pop(0) + matches = re.search(r"([\d\.]+)", first_line) + if matches: + version = matches.group(0) + return Version.from_str(version) + return None + + def pull(self, *args) -> str: + stdout, _ = subp([*self.cmd_pull, *args], env=self.env) + return stdout + + def check_deps(self): + if not self.is_installed(): + raise ValueError("command: ansible is not installed") + + @abc.abstractmethod + def is_installed(self): + pass + + @abc.abstractmethod + def install(self, pkg_name: str): + pass + + +class AnsiblePullPip(AnsiblePull): + def __init__(self): + self.cmd_pull = ["ansible-pull"] + self.cmd_version = ["ansible-pull", "--version"] + self.env["PATH"] = ":".join([self.env["PATH"], "/root/.local/bin/"]) + + def install(self, pkg_name: str): + """should cloud-init grow an interface for non-distro package + managers? this seems reusable + """ + if not self.is_installed(): + subp(["python3", "-m", "pip", "install", "--user", pkg_name]) + + def is_installed(self) -> bool: + stdout, _ = subp(["python3", "-m", "pip", "list"]) + return "ansible" in stdout + + +class AnsiblePullDistro(AnsiblePull): + def __init__(self, distro): + self.cmd_pull = ["ansible-pull"] + self.cmd_version = ["ansible-pull", "--version"] + self.distro = distro + + def install(self, pkg_name: str): + if not self.is_installed(): + self.distro.install_packages(pkg_name) + + def is_installed(self) -> bool: + return bool(which("ansible")) + + +def handle(name: str, cfg: dict, cloud: Cloud, _, __): + ansible_cfg: dict = cfg.get("ansible", {}) + if ansible_cfg: + validate_config(ansible_cfg) + install = ansible_cfg["install-method"] + pull_cfg = ansible_cfg.get("pull") + if pull_cfg: + ansible: AnsiblePull + if install == "pip": + ansible = AnsiblePullPip() + else: + ansible = AnsiblePullDistro(cloud.distro) + ansible.install(ansible_cfg["package-name"]) + ansible.check_deps() + run_ansible_pull(ansible, deepcopy(pull_cfg)) + + +def validate_config(cfg: dict): + required_keys = { + "install-method", + "package-name", + "pull/url", + "pull/playbook-name", + } + for key in required_keys: + if not get_cfg_by_path(cfg, key): + raise ValueError(f"Invalid value config key: '{key}'") + + install = cfg["install-method"] + if install not in ("pip", "distro"): + raise ValueError("Invalid install method {install}") + + +def filter_args(cfg: dict) -> dict: + """remove boolean false values""" + return {key: value for (key, value) in cfg.items() if value is not False} + + +def run_ansible_pull(pull: AnsiblePull, cfg: dict): + playbook_name: str = cfg.pop("playbook-name") + + v = pull.get_version() + if not v: + LOG.warning("Cannot parse ansible version") + elif v < Version(2, 7, 0): + # diff was added in commit edaa0b52450ade9b86b5f63097ce18ebb147f46f + if cfg.get("diff"): + raise ValueError( + f"Ansible version {v.major}.{v.minor}.{v.patch}" + "doesn't support --diff flag, exiting." + ) + stdout = pull.pull( + *[ + f"--{key}={value}" if value is not True else f"--{key}" + for key, value in filter_args(cfg).items() + ], + playbook_name, + ) + if stdout: + sys.stdout.write(f"{stdout}") diff --git a/cloudinit/config/cc_apk_configure.py b/cloudinit/config/cc_apk_configure.py index 0952c971e..0fd7d229d 100644 --- a/cloudinit/config/cc_apk_configure.py +++ b/cloudinit/config/cc_apk_configure.py @@ -100,6 +100,7 @@ ), ], "frequency": frequency, + "activate_by_schema_keys": ["apk_repos"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 7ca501943..9d39c918f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -118,6 +118,7 @@ ) ], "frequency": frequency, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -756,7 +757,7 @@ def search_for_mirror_dns(configured, mirrortype, cfg, cloud): raise ValueError("unknown mirror type") # if we have a fqdn, then search its domain portion first - (_, fqdn) = util.get_hostname_fqdn(cfg, cloud) + fqdn = util.get_hostname_fqdn(cfg, cloud).fqdn mydom = ".".join(fqdn.split(".")[1:]) if mydom: doms.append(".%s" % mydom) diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index 901633d32..82a8e6e02 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -50,6 +50,7 @@ "apt_pipelining: os", "apt_pipelining: 3", ], + "activate_by_schema_keys": ["apt_pipelining"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index bd14aedec..4ee798592 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -54,6 +54,7 @@ ) ], "frequency": PER_ALWAYS, + "activate_by_schema_keys": ["bootcmd"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index fbc20410b..e48fce340 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -43,6 +43,7 @@ "byobu_by_default: enable-user", "byobu_by_default: disable-system", ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 6084cb4cf..6c9c7ab43 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -66,6 +66,7 @@ """ ) ], + "activate_by_schema_keys": ["ca_certs", "ca-certs"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fdb3a6e37..5ab2b401b 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -71,19 +71,20 @@ "encrypted_data_bag_secret", ] ) -CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) -CHEF_RB_TPL_KEYS.extend( - [ - "server_url", - "node_name", - "environment", - "validation_name", - "chef_license", - ] +CHEF_RB_TPL_KEYS = frozenset( + itertools.chain( + CHEF_RB_TPL_DEFAULTS.keys(), + CHEF_RB_TPL_BOOL_KEYS, + CHEF_RB_TPL_PATH_KEYS, + [ + "server_url", + "node_name", + "environment", + "validation_name", + "chef_license", + ], + ) ) -CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = "/etc/chef/client.rb" CHEF_EXEC_PATH = "/usr/bin/chef-client" CHEF_EXEC_DEF_ARGS = tuple(["-d", "-i", "1800", "-s", "20"]) @@ -135,6 +136,7 @@ ) ], "frequency": frequency, + "activate_by_schema_keys": ["chef"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py deleted file mode 100644 index c51818c3e..000000000 --- a/cloudinit/config/cc_debug.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (C) 2013 Yahoo! Inc. -# -# This file is part of cloud-init. See LICENSE file for license information. - -"""Debug: Helper to debug cloud-init *internal* datastructures.""" - -import copy -from io import StringIO -from textwrap import dedent - -from cloudinit import safeyaml, type_utils, util -from cloudinit.config.schema import MetaSchema, get_meta_doc -from cloudinit.distros import ALL_DISTROS -from cloudinit.settings import PER_INSTANCE - -SKIP_KEYS = frozenset(["log_cfgs"]) - -MODULE_DESCRIPTION = """\ -This module will enable for outputting various internal information that -cloud-init sources provide to either a file or to the output console/log -location that this cloud-init has been configured with when running. - -.. note:: - Log configurations are not output. -""" - -meta: MetaSchema = { - "id": "cc_debug", - "name": "Debug", - "title": "Helper to debug cloud-init *internal* datastructures", - "description": MODULE_DESCRIPTION, - "distros": [ALL_DISTROS], - "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - debug: - verbose: true - output: /tmp/my_debug.log - """ - ) - ], -} - -__doc__ = get_meta_doc(meta) - - -def _make_header(text): - header = StringIO() - header.write("-" * 80) - header.write("\n") - header.write(text.center(80, " ")) - header.write("\n") - header.write("-" * 80) - header.write("\n") - return header.getvalue() - - -def _dumps(obj): - text = safeyaml.dumps(obj, explicit_start=False, explicit_end=False) - return text.rstrip() - - -def handle(name, cfg, cloud, log, args): - """Handler method activated by cloud-init.""" - verbose = util.get_cfg_by_path(cfg, ("debug", "verbose"), default=True) - if args: - # if args are provided (from cmdline) then explicitly set verbose - out_file = args[0] - verbose = True - else: - out_file = util.get_cfg_by_path(cfg, ("debug", "output")) - - if not verbose: - log.debug("Skipping module named %s, verbose printing disabled", name) - return - # Clean out some keys that we just don't care about showing... - dump_cfg = copy.deepcopy(cfg) - for k in SKIP_KEYS: - dump_cfg.pop(k, None) - all_keys = list(dump_cfg) - for k in all_keys: - if k.startswith("_"): - dump_cfg.pop(k, None) - # Now dump it... - to_print = StringIO() - to_print.write(_make_header("Config")) - to_print.write(_dumps(dump_cfg)) - to_print.write("\n") - to_print.write(_make_header("MetaData")) - to_print.write(_dumps(cloud.datasource.metadata)) - to_print.write("\n") - to_print.write(_make_header("Misc")) - to_print.write( - "Datasource: %s\n" % (type_utils.obj_name(cloud.datasource)) - ) - to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) - to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) - to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) - to_print.write("Locale: %s\n" % (cloud.get_locale())) - to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) - contents = to_print.getvalue() - content_to_file = [] - for line in contents.splitlines(): - line = "ci-info: %s\n" % (line) - content_to_file.append(line) - if out_file: - util.write_file(out_file, "".join(content_to_file), 0o644, "w") - else: - util.multi_log("".join(content_to_file), console=True, stderr=False) - - -# vi: ts=4 expandtab diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 88cc28e21..a7832e258 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -31,6 +31,7 @@ "distros": [ALL_DISTROS], "frequency": PER_ALWAYS, "examples": ["disable_ec2_metadata: true"], + "activate_by_schema_keys": ["disable_ec2_metadata"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index ee05ea878..182e9401c 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -81,6 +81,7 @@ """ ) ], + "activate_by_schema_keys": ["disk_setup", "fs_setup"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 57c762a12..094baa091 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -49,6 +49,7 @@ """ ) ], + "activate_by_schema_keys": ["fan"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 89be520e6..c44f021f0 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -45,6 +45,7 @@ """ ) ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 14a2c0b88..e3ba0e9a5 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -81,6 +81,7 @@ """ ), ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -346,7 +347,7 @@ def is_encrypted(blockdev, partition) -> bool: def get_underlying_partition(blockdev): command = ["dmsetup", "deps", "--options=devname", blockdev] - dep: str = subp.subp(command)[0] # type: ignore + dep: str = subp.subp(command)[0] # pyright: ignore # Returned result should look something like: # 1 dependencies : (vdb1) if not dep.startswith("1 depend"): diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index c23e40f5b..f2fa69850 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -49,6 +49,7 @@ """ ) ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -133,11 +134,6 @@ def handle(name, cfg, _cloud, log, _args): if idevs_empty is None: idevs_empty = not idevs elif not isinstance(idevs_empty, bool): - log.warning( - "DEPRECATED: grub_dpkg: grub-pc/install_devices_empty value of " - f"'{idevs_empty}' is not boolean. Use of non-boolean values " - "will be removed in a future version of cloud-init." - ) idevs_empty = util.translate_bool(idevs_empty) idevs_empty = str(idevs_empty).lower() diff --git a/cloudinit/config/cc_install_hotplug.py b/cloudinit/config/cc_install_hotplug.py index a3668232e..e29b58b9d 100644 --- a/cloudinit/config/cc_install_hotplug.py +++ b/cloudinit/config/cc_install_hotplug.py @@ -49,6 +49,7 @@ """ ), ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_keyboard.py b/cloudinit/config/cc_keyboard.py index 211cb0156..e44b86483 100644 --- a/cloudinit/config/cc_keyboard.py +++ b/cloudinit/config/cc_keyboard.py @@ -18,14 +18,16 @@ DEFAULT_KEYBOARD_MODEL = "pc105" -distros = distros.Distro.expand_osfamily(["arch", "debian", "redhat", "suse"]) +supported_distros = distros.Distro.expand_osfamily( + ["arch", "debian", "redhat", "suse"] +) meta: MetaSchema = { "id": "cc_keyboard", "name": "Keyboard", "title": "Set keyboard layout", "description": "Handle keyboard configuration.", - "distros": distros, + "distros": supported_distros, "examples": [ dedent( """\ @@ -46,6 +48,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["keyboard"], } diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index dd8b92fe2..115df520a 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -61,6 +61,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index ede09bd9a..2607b866e 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -91,6 +91,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["landscape"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 6a31933e2..dd7fda38f 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -42,6 +42,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 847a7c3c7..490533c03 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -7,10 +7,13 @@ """LXD: configure lxd with ``lxd init`` and optionally lxd-bridge""" import os +from logging import Logger from textwrap import dedent +from typing import List, Tuple from cloudinit import log as logging from cloudinit import subp, util +from cloudinit.cloud import Cloud from cloudinit.config.schema import MetaSchema, get_meta_doc from cloudinit.settings import PER_INSTANCE @@ -22,9 +25,9 @@ MODULE_DESCRIPTION = """\ This module configures lxd with user specified options using ``lxd init``. If lxd is not present on the system but lxd configuration is provided, then -lxd will be installed. If the selected storage backend is zfs, then zfs will -be installed if missing. If network bridge configuration is provided, then -lxd-bridge will be configured accordingly. +lxd will be installed. If the selected storage backend userspace utility is +not installed, it will be installed. If network bridge configuration is +provided, then lxd-bridge will be configured accordingly. """ distros = ["ubuntu"] @@ -55,6 +58,7 @@ storage_create_loop: 10 bridge: mode: new + mtu: 1500 name: lxdbr0 ipv4_address: 10.0.8.1 ipv4_netmask: 24 @@ -70,12 +74,13 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["lxd"], } __doc__ = get_meta_doc(meta) -def handle(name, cfg, cloud, log, args): +def handle(name, cfg, cloud: Cloud, log: Logger, args): # Get config lxd_cfg = cfg.get("lxd") if not lxd_cfg: @@ -105,15 +110,7 @@ def handle(name, cfg, cloud, log, args): type(bridge_cfg), ) bridge_cfg = {} - - # Install the needed packages - packages = [] - if not subp.which("lxd"): - packages.append("lxd") - - if init_cfg.get("storage_backend") == "zfs" and not subp.which("zfs"): - packages.append("zfsutils-linux") - + packages = get_required_packages(init_cfg) if len(packages): try: cloud.distro.install_packages(packages) @@ -123,7 +120,10 @@ def handle(name, cfg, cloud, log, args): # Set up lxd if init config is given if init_cfg: - init_keys = ( + + # type is known, number of elements is not + # in the case of the ubuntu+lvm backend workaround + init_keys: Tuple[str, ...] = ( "network_address", "network_port", "storage_backend", @@ -132,7 +132,36 @@ def handle(name, cfg, cloud, log, args): "storage_pool", "trust_password", ) + subp.subp(["lxd", "waitready", "--timeout=300"]) + + # Bug https://bugs.launchpad.net/ubuntu/+source/linux-kvm/+bug/1982780 + kernel = util.system_info()["uname"][2] + if init_cfg["storage_backend"] == "lvm" and not os.path.exists( + f"/lib/modules/{kernel}/kernel/drivers/md/dm-thin-pool.ko" + ): + log.warning( + "cloud-init doesn't use thinpool by default on Ubuntu due to " + "LP #1982780. This behavior will change in the future.", + ) + subp.subp( + [ + "lxc", + "storage", + "create", + "default", + "lvm", + "lvm.use_thinpool=false", + ] + ) + + # Since we're manually setting use_thinpool=false + # filter it from the lxd init commands, don't configure + # storage twice + init_keys = tuple( + key for key in init_keys if key != "storage_backend" + ) + cmd = ["lxd", "init", "--auto"] for k in init_keys: if init_cfg.get(k): @@ -298,6 +327,12 @@ def bridge_to_cmd(bridge_cfg): if bridge_cfg.get("domain"): cmd_create.append("dns.domain=%s" % bridge_cfg.get("domain")) + # if the default schema value is passed (-1) don't pass arguments + # to LXD. Use LXD defaults unless user manually sets a number + mtu = bridge_cfg.get("mtu", -1) + if mtu != -1: + cmd_create.append(f"bridge.mtu={mtu}") + return cmd_create, cmd_attach @@ -350,4 +385,20 @@ def maybe_cleanup_default( LOG.debug(msg, nic_name, profile, fail_assume_enoent) -# vi: ts=4 expandtab +def get_required_packages(cfg: dict) -> List[str]: + """identify required packages for install""" + packages = [] + if not subp.which("lxd"): + packages.append("lxd") + + # binary for pool creation must be available for the requested backend: + # zfs, lvcreate, mkfs.btrfs + storage: str = cfg.get("storage_backend", "") + if storage: + if storage == "zfs" and not subp.which("zfs"): + packages.append("zfsutils-linux") + if storage == "lvm" and not subp.which("lvcreate"): + packages.append("lvm2") + if storage == "btrfs" and not subp.which("mkfs.btrfs"): + packages.append("btrfs-progs") + return packages diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 33f7556da..f4fd456e0 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -82,6 +82,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["mcollective"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index 6aed54b39..f1cd788a6 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -32,6 +32,7 @@ "distros": distros, "examples": ["# Do not migrate cloud-init semaphores\nmigrate: false\n"], "frequency": frequency, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1d05c9b98..843ea5ebe 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -9,6 +9,7 @@ """Mounts: Configure mount points and swap files""" import logging +import math import os import re from string import whitespace @@ -97,6 +98,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -110,6 +112,8 @@ WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" MNT_COMMENT = "comment=cloudconfig" +MB = 2**20 +GB = 2**30 LOG = logging.getLogger(__name__) @@ -210,13 +214,12 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): if memsize is None: memsize = util.read_meminfo()["total"] - GB = 2**30 - sugg_max = 8 * GB + sugg_max = memsize * 2 info = {"avail": "na", "max_in": maxsize, "mem": memsize} if fsys is None and maxsize is None: - # set max to 8GB default if no filesystem given + # set max to default if no filesystem given maxsize = sugg_max elif fsys: statvfs = os.statvfs(fsys) @@ -234,35 +237,17 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): info["max"] = maxsize - formulas = [ - # < 1G: swap = double memory - (1 * GB, lambda x: x * 2), - # < 2G: swap = 2G - (2 * GB, lambda x: 2 * GB), - # < 4G: swap = memory - (4 * GB, lambda x: x), - # < 16G: 4G - (16 * GB, lambda x: 4 * GB), - # < 64G: 1/2 M up to max - (64 * GB, lambda x: x / 2), - ] - - size = None - for top, func in formulas: - if memsize <= top: - size = min(func(memsize), maxsize) - # if less than 1/2 memory and not much, return 0 - if size < (memsize / 2) and size < 4 * GB: - size = 0 - break - break + if memsize < 4 * GB: + minsize = memsize + elif memsize < 16 * GB: + minsize = 4 * GB + else: + minsize = round(math.sqrt(memsize / GB)) * GB - if size is not None: - size = maxsize + size = min(minsize, maxsize) info["size"] = size - MB = 2**20 pinfo = {} for k, v in info.items(): if isinstance(v, int): diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3bc1d3031..7974f5b26 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -30,6 +30,7 @@ "fedora", "miraclelinux", "openEuler", + "openmandriva", "opensuse", "photon", "rhel", @@ -87,11 +88,31 @@ "service_name": "ntpd", }, }, + "centos": { + "ntp": { + "service_name": "ntpd", + }, + "chrony": { + "service_name": "chronyd", + }, + }, "debian": { "chrony": { "confpath": "/etc/chrony/chrony.conf", }, }, + "openmandriva": { + "chrony": { + "service_name": "chronyd", + }, + "ntp": { + "confpath": "/etc/ntp.conf", + "service_name": "ntpd", + }, + "systemd-timesyncd": { + "check_exe": "/lib/systemd/systemd-timesyncd", + }, + }, "opensuse": { "chrony": { "service_name": "chronyd", @@ -205,6 +226,7 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["ntp"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 5198305e4..a8a3e9ff5 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -47,6 +47,13 @@ """ ) ], + "activate_by_schema_keys": [ + "apt_update", + "package_update", + "apt_upgrade", + "package_upgrade", + "packages", + ], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index bae54134a..dee30e961 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -88,6 +88,7 @@ """ ), ], + "activate_by_schema_keys": ["phone_home"], } __doc__ = get_meta_doc(meta) @@ -129,7 +130,7 @@ def handle(name, cfg, cloud, log, args): post_list = ph_cfg.get("post", "all") tries = ph_cfg.get("tries") try: - tries = int(tries) # type: ignore + tries = int(tries) # pyright: ignore except (ValueError, TypeError): tries = 10 util.logexc( @@ -143,8 +144,8 @@ def handle(name, cfg, cloud, log, args): all_keys = { "instance_id": cloud.get_instance_id(), - "hostname": cloud.get_hostname(), - "fqdn": cloud.get_hostname(fqdn=True), + "hostname": cloud.get_hostname().hostname, + "fqdn": cloud.get_hostname(fqdn=True).hostname, } pubkeys = { diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 7fc4e5ca9..39459bfef 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -72,6 +72,7 @@ """ ), ], + "activate_by_schema_keys": ["power_state"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index c0b073b5b..14467e364 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -98,6 +98,7 @@ """ ), ], + "activate_by_schema_keys": ["puppet"], } __doc__ = get_meta_doc(meta) @@ -257,7 +258,6 @@ def handle(name, cfg, cloud, log, _args): # (TODO(harlowja) is this really needed??) cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = "\n".join(cleaned_lines) - # Move to puppet_config.read_file when dropping py2.7 puppet_config.read_file( StringIO(cleaned_contents), source=p_constants.conf_path ) diff --git a/cloudinit/config/cc_refresh_rmc_and_interface.py b/cloudinit/config/cc_refresh_rmc_and_interface.py index 3ed5612b9..180a8873f 100644 --- a/cloudinit/config/cc_refresh_rmc_and_interface.py +++ b/cloudinit/config/cc_refresh_rmc_and_interface.py @@ -42,6 +42,7 @@ "distros": [ALL_DISTROS], "frequency": PER_ALWAYS, "examples": [], + "activate_by_schema_keys": [], } # This module is undocumented in our schema docs diff --git a/cloudinit/config/cc_reset_rmc.py b/cloudinit/config/cc_reset_rmc.py index 57f024efb..9766c3a40 100644 --- a/cloudinit/config/cc_reset_rmc.py +++ b/cloudinit/config/cc_reset_rmc.py @@ -43,6 +43,7 @@ "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [], + "activate_by_schema_keys": [], } # This module is undocumented in our schema docs diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 39da1b5a8..3372208f3 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -42,6 +42,7 @@ "resize_rootfs: noblock # runs resize operation in the background", ], "frequency": PER_ALWAYS, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index bbf680792..545b22c3b 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -75,6 +75,7 @@ """ ) ], + "activate_by_schema_keys": ["manage_resolv_conf"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index b742cb951..9dfe6a386 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -71,6 +71,7 @@ """ ), ], + "activate_by_schema_keys": ["rh_subscription"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index c1b0f8bd3..5ebf359f3 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -44,6 +44,7 @@ "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 57b8aa621..5484691b3 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -60,6 +60,7 @@ """ ), ], + "activate_by_schema_keys": ["rsyslog"], } __doc__ = get_meta_doc(meta) @@ -67,7 +68,7 @@ DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" DEF_RELOAD = "auto" -DEF_REMOTES = {} +DEF_REMOTES: dict = {} KEYNAME_CONFIGS = "configs" KEYNAME_FILENAME = "config_filename" @@ -113,7 +114,7 @@ def load_config(cfg: dict) -> dict: if KEYNAME_LEGACY_DIR in cfg: mycfg[KEYNAME_DIR] = cfg[KEYNAME_LEGACY_DIR] - fillup = ( + fillup: tuple = ( (KEYNAME_CONFIGS, [], list), (KEYNAME_DIR, DEF_DIR, str), (KEYNAME_FILENAME, DEF_FILENAME, str), diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 7c614f575..60e53298f 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -24,11 +24,14 @@ MODULE_DESCRIPTION = """\ -Run arbitrary commands at a rc.local like level with output to the -console. Each item can be either a list or a string. If the item is a -list, it will be properly quoted. Each item is written to -``/var/lib/cloud/instance/runcmd`` to be later interpreted using -``sh``. +Run arbitrary commands at a rc.local like time-frame with output to the +console. Each item can be either a list or a string. The item type affects +how it is executed: + + +* If the item is a string, it will be interpreted by ``sh``. +* If the item is a list, the items will be executed as if passed to execve(3) + (with the first arg as the command). Note that the ``runcmd`` module only writes the script to be run later. The module that actually runs the script is ``scripts-user`` @@ -64,6 +67,7 @@ """ ) ], + "activate_by_schema_keys": ["runcmd"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index df9d4205b..ebab4e30a 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -58,6 +58,7 @@ """ ) ], + "activate_by_schema_keys": ["salt_minion"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index aa311d595..408c3bfd4 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -30,6 +30,7 @@ "distros": [ALL_DISTROS], "frequency": frequency, "examples": [], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index 1fb407175..c1360ae6c 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -31,6 +31,7 @@ "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index d9f406b7d..baf2214e2 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -30,6 +30,7 @@ "distros": [ALL_DISTROS], "frequency": frequency, "examples": [], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 85375dac3..ffe610fd2 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -31,6 +31,7 @@ "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index 894404f8e..8dc99e1ec 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -52,6 +52,7 @@ """ ), ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index b0ffdd157..f829eaf45 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -69,6 +69,7 @@ """ ), ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index e938e4aee..6cf593fac 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -14,9 +14,9 @@ from cloudinit.atomic_helper import write_json from cloudinit.config.schema import MetaSchema, get_meta_doc from cloudinit.distros import ALL_DISTROS -from cloudinit.settings import PER_ALWAYS +from cloudinit.settings import PER_INSTANCE -frequency = PER_ALWAYS +frequency = PER_INSTANCE MODULE_DESCRIPTION = """\ This module handles setting the system hostname and fully qualified domain name (FQDN). If ``preserve_hostname`` is set, then the hostname will not be @@ -40,7 +40,7 @@ This will occur on datasources like nocloud and ovf where metadata and user data are available locally. This ensures that the desired hostname is applied -before any DHCP requests are preformed on these platforms where dynamic DNS is +before any DHCP requests are performed on these platforms where dynamic DNS is based on initial hostname. """ @@ -61,6 +61,7 @@ """ ), ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -90,7 +91,7 @@ def handle(name, cfg, cloud, log, _args): if hostname_fqdn is not None: cloud.distro.set_option("prefer_fqdn_over_hostname", hostname_fqdn) - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn, is_default) = util.get_hostname_fqdn(cfg, cloud) if fqdn[-1] == '.': fqdn = fqdn[:-1] # Check for previous successful invocation of set-hostname @@ -110,6 +111,10 @@ def handle(name, cfg, cloud, log, _args): if not hostname_changed: log.debug("No hostname changes. Skipping set-hostname") return + if is_default and hostname == "localhost": + # https://github.com/systemd/systemd/commit/d39079fcaa05e23540d2b1f0270fa31c22a7e9f1 + log.debug("Hostname is localhost. Let other services handle this.") + return log.debug("Setting the hostname to %s (%s)", fqdn, hostname) try: cloud.distro.set_hostname(hostname, fqdn) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 3c8b378bf..fa7de9444 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -8,11 +8,15 @@ """Set Passwords: Set user passwords and enable/disable SSH password auth""" import re +from logging import Logger from string import ascii_letters, digits from textwrap import dedent +from typing import List +from cloudinit import features from cloudinit import log as logging from cloudinit import subp, util +from cloudinit.cloud import Cloud from cloudinit.config.schema import MetaSchema, get_meta_doc from cloudinit.distros import ALL_DISTROS, Distro, ug_util from cloudinit.settings import PER_INSTANCE @@ -26,13 +30,19 @@ to accept password authentication. The ``chpasswd`` config key accepts a dictionary containing either or both of -``list`` and ``expire``. The ``list`` key is used to assign a password to a -to a corresponding pre-existing user. The ``expire`` key is used to set -whether to expire all user passwords such that a password will need to be reset -on the user's next login. +``users`` and ``expire``. The ``users`` key is used to assign a password to a +corresponding pre-existing user. The ``expire`` key is used to set +whether to expire all user passwords specified by this module, +such that a password will need to be reset on the user's next login. + +.. note:: + Prior to cloud-init 22.3, the ``expire`` key only applies to plain text + (including ``RANDOM``) passwords. Post 22.3, the ``expire`` key applies to + both plain text and hashed passwords. ``password`` config key is used to set the default user's password. It is -ignored if the ``chpasswd`` ``list`` is used. +ignored if the ``chpasswd`` ``users`` is used. Note: the ``list`` keyword is +deprecated in favor of ``users``. """ meta: MetaSchema = { @@ -56,19 +66,24 @@ # Disable ssh password authentication # Don't require users to change their passwords on next login # Set the password for user1 to be 'password1' (OS does hashing) - # Set the password for user2 to be a randomly generated password, + # Set the password for user2 to a pre-hashed password + # Set the password for user3 to be a randomly generated password, # which will be written to the system console - # Set the password for user3 to a pre-hashed password ssh_pwauth: false chpasswd: expire: false - list: - - user1:password1 - - user2:RANDOM - - user3:$6$rounds=4096$5DJ8a9WMTEzIo5J4$Yms6imfeBvf3Yfu84mQBerh18l7OR1Wm1BJXZqFSpJ6BVas0AYJqIjP7czkOaAZHZi1kxQ5Y1IhgWN8K9NgxR1 + users: + - name: user1 + password: password1 + type: text + - name: user2 + password: $6$rounds=4096$5DJ8a9WMTEzIo5J4$Yms6imfeBvf3Yfu84mQBerh18l7OR1Wm1BJXZqFSpJ6BVas0AYJqIjP7czkOaAZHZi1kxQ5Y1IhgWN8K9NgxR1 + - name: user3 + type: RANDOM """ # noqa ), ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -79,6 +94,19 @@ PW_SET = "".join([x for x in ascii_letters + digits if x not in "loLOI01"]) +def get_users_by_type(users_list: list, pw_type: str) -> list: + """either password or type: RANDOM is required, user is always required""" + return ( + [] + if not users_list + else [ + (item["name"], item.get("password", "RANDOM")) + for item in users_list + if item.get("type", "hash") == pw_type + ] + ) + + def handle_ssh_pwauth(pw_auth, distro: Distro): """Apply sshd PasswordAuthentication changes. @@ -143,7 +171,7 @@ def handle_ssh_pwauth(pw_auth, distro: Distro): elif util.is_false(pw_auth): cfg_val = "no" else: - bmsg = "Leaving SSH config '%s' unchanged." % cfg_name + bmsg = f"Leaving SSH config '{cfg_name}' unchanged." if pw_auth is None or pw_auth.lower() == "unchanged": LOG.debug("%s ssh_pwauth=%s", bmsg, pw_auth) else: @@ -162,7 +190,8 @@ def handle_ssh_pwauth(pw_auth, distro: Distro): LOG.debug("Not restarting SSH service: service is stopped.") -def handle(_name, cfg, cloud, log, args): +def handle(_name, cfg: dict, cloud: Cloud, log: Logger, args: list): + distro: Distro = cloud.distro if args: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] @@ -172,11 +201,16 @@ def handle(_name, cfg, cloud, log, args): password = util.get_cfg_option_str(cfg, "password", None) expire = True - plist = None + plist: List = [] + users_list: List = [] if "chpasswd" in cfg: chfg = cfg["chpasswd"] + users_list = util.get_cfg_option_list(chfg, "users", default=[]) if "list" in chfg and chfg["list"]: + log.warning( + "DEPRECATION: key 'lists' is now deprecated. Use 'users'." + ) if isinstance(chfg["list"], list): log.debug("Handling input for chpasswd as list.") plist = util.get_cfg_option_list(chfg, "list", plist) @@ -187,14 +221,14 @@ def handle(_name, cfg, cloud, log, args): "cloud-init. Use the list format instead." ) log.debug("Handling input for chpasswd as multiline string.") - plist = util.get_cfg_option_str(chfg, "list", plist) - if plist: - plist = plist.splitlines() + multiline = util.get_cfg_option_str(chfg, "list") + if multiline: + plist = multiline.splitlines() expire = util.get_cfg_option_bool(chfg, "expire", expire) - if not plist and password: - (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) + if not (users_list or plist) and password: + (users, _groups) = ug_util.normalize_users_groups(cfg, distro) (user, _user_config) = ug_util.extract_default(users) if user: plist = ["%s:%s" % (user, password)] @@ -202,19 +236,32 @@ def handle(_name, cfg, cloud, log, args): log.warning("No default or defined user to change password for.") errors = [] - if plist: - plist_in = [] - hashed_plist_in = [] - hashed_users = [] + if plist or users_list: + # This section is for parsing the data that arrives in the form of + # chpasswd: + # users: + plist_in = get_users_by_type(users_list, "text") + users = [user for user, _ in plist_in] + hashed_plist_in = get_users_by_type(users_list, "hash") + hashed_users = [user for user, _ in hashed_plist_in] randlist = [] - users = [] - # N.B. This regex is included in the documentation (i.e. the module + for user, _ in get_users_by_type(users_list, "RANDOM"): + password = rand_user_password() + users.append(user) + plist_in.append((user, password)) + randlist.append(f"{user}:{password}") + + # This for loop is for parsing the data that arrives in the deprecated + # form of + # chpasswd: + # list: + # N.B. This regex is included in the documentation (i.e. the schema # docstring), so any changes to it should be reflected there. prog = re.compile(r"\$(1|2a|2y|5|6)(\$.+){2}") for line in plist: u, p = line.split(":", 1) if prog.match(p) is not None and ":" not in p: - hashed_plist_in.append(line) + hashed_plist_in.append((u, p)) hashed_users.append(u) else: # in this else branch, we potentially change the password @@ -222,24 +269,22 @@ def handle(_name, cfg, cloud, log, args): if p == "R" or p == "RANDOM": p = rand_user_password() randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) + plist_in.append((u, p)) users.append(u) - ch_in = "\n".join(plist_in) + "\n" if users: try: log.debug("Changing password for %s:", users) - chpasswd(cloud.distro, ch_in) + distro.chpasswd(plist_in, hashed=False) except Exception as e: errors.append(e) util.logexc( log, "Failed to set passwords with chpasswd for %s", users ) - hashed_ch_in = "\n".join(hashed_plist_in) + "\n" if hashed_users: try: log.debug("Setting hashed password for %s:", hashed_users) - chpasswd(cloud.distro, hashed_ch_in, hashed=True) + distro.chpasswd(hashed_plist_in, hashed=True) except Exception as e: errors.append(e) util.logexc( @@ -258,10 +303,13 @@ def handle(_name, cfg, cloud, log, args): ) if expire: + users_to_expire = users + if features.EXPIRE_APPLIES_TO_HASHED_USERS: + users_to_expire += hashed_users expired_users = [] - for u in users: + for u in users_to_expire: try: - cloud.distro.expire_passwd(u) + distro.expire_passwd(u) expired_users.append(u) except Exception as e: errors.append(e) @@ -269,7 +317,7 @@ def handle(_name, cfg, cloud, log, args): if expired_users: log.debug("Expired passwords for: %s users", expired_users) - handle_ssh_pwauth(cfg.get("ssh_pwauth"), cloud.distro) + handle_ssh_pwauth(cfg.get("ssh_pwauth"), distro) if len(errors): log.debug("%s errors occurred, re-raising the last one", len(errors)) @@ -278,16 +326,3 @@ def handle(_name, cfg, cloud, log, args): def rand_user_password(pwlen=20): return util.rand_str(pwlen, select_from=PW_SET) - - -def chpasswd(distro, plist_in, hashed=False): - if util.is_BSD(): - for pentry in plist_in.splitlines(): - u, p = pentry.split(":") - distro.set_passwd(u, p, hashed=hashed) - else: - cmd = ["chpasswd"] + (["-e"] if hashed else []) - subp.subp(cmd, plist_in) - - -# vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 41a6adf94..7bf22a52f 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -4,6 +4,7 @@ """Snap: Install, configure and manage snapd and snap packages.""" +import os import sys from textwrap import dedent @@ -104,16 +105,16 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["snap"], } __doc__ = get_meta_doc(meta) SNAP_CMD = "snap" -ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" -def add_assertions(assertions): +def add_assertions(assertions, assertions_file): """Import list of assertions. Import assertions by concatenating each assertion into a @@ -133,14 +134,14 @@ def add_assertions(assertions): ) ) - snap_cmd = [SNAP_CMD, "ack"] + snap_cmd = [SNAP_CMD, "ack", assertions_file] combined = "\n".join(assertions) for asrt in assertions: LOG.debug("Snap acking: %s", asrt.split("\n")[0:2]) - util.write_file(ASSERTIONS_FILE, combined.encode("utf-8")) - subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + util.write_file(assertions_file, combined.encode("utf-8")) + subp.subp(snap_cmd, capture=True) def run_commands(commands): @@ -190,7 +191,10 @@ def handle(name, cfg, cloud, log, args): ) return - add_assertions(cfgin.get("assertions", [])) + add_assertions( + cfgin.get("assertions", []), + os.path.join(cloud.paths.get_ipath_cur(), "snapd.assertions"), + ) run_commands(cfgin.get("commands", [])) diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py index 6820a816a..991aa5edd 100644 --- a/cloudinit/config/cc_spacewalk.py +++ b/cloudinit/config/cc_spacewalk.py @@ -34,6 +34,7 @@ """ ) ], + "activate_by_schema_keys": ["spacewalk"], } __doc__ = get_meta_doc(meta) @@ -99,7 +100,7 @@ def handle(name, cfg, cloud, log, _args): if not is_registered(): do_register( spacewalk_server, - cloud.datasource.get_hostname(fqdn=True), + cloud.datasource.get_hostname(fqdn=True).hostname, proxy=cfg.get("proxy"), log=log, activation_key=cfg.get("activation_key"), diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 33c1fd0cc..ad4fcf80c 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -165,6 +165,7 @@ """ # noqa: E501 ) ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -213,7 +214,7 @@ def handle(_name, cfg, cloud: Cloud, log: Logger, _args): reason = "unsupported" else: reason = "unrecognized" - log.warning("Skipping %s ssh_keys" ' entry: "%s"', reason, key) + log.warning('Skipping %s ssh_keys entry: "%s"', reason, key) continue tgt_fn = CONFIG_KEY_TO_FILE[key][0] tgt_perms = CONFIG_KEY_TO_FILE[key][1] diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index db5c14549..40fb4ce59 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -31,6 +31,7 @@ "no_ssh_fingerprints: true", "authkey_hash: sha512", ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 6a15895d3..358c571f2 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -43,6 +43,7 @@ """ ) ], + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -57,7 +58,7 @@ def handle(_name, cfg, cloud, log, args): ) return elif not subp.which(SSH_IMPORT_ID_BINARY): - log.warn( + log.warning( "ssh-import-id is not installed, but module ssh_import_id is " "configured. Skipping module." ) diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index 47da2d065..b9df31af1 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -26,6 +26,7 @@ "examples": [ "timezone: US/Eastern", ], + "activate_by_schema_keys": ["timezone"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 57763c318..c05d62974 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -2,7 +2,9 @@ """ubuntu_advantage: Configure Ubuntu Advantage support services""" +import re from textwrap import dedent +from urllib.parse import urlparse from cloudinit import log as logging from cloudinit import subp, util @@ -23,8 +25,8 @@ enable or disable support services such as Livepatch, ESM, FIPS and FIPS Updates. When attaching a machine to Ubuntu Advantage, one can also specify services to enable. When the 'enable' - list is present, any named service will be enabled and all absent - services will remain disabled. + list is present, any named service will supplement the contract-default + enabled services. Note that when enabling FIPS or FIPS updates you will need to schedule a reboot to ensure the machine is running the FIPS-compliant kernel. @@ -69,8 +71,26 @@ - fips """ ), + dedent( + """\ + # Set a http(s) proxy before attaching the machine to an + # Ubuntu Advantage support contract and enabling the FIPS service. + ubuntu_advantage: + token: + config: + http_proxy: 'http://some-proxy:8088' + https_proxy: 'https://some-proxy:8088' + global_apt_https_proxy: 'http://some-global-apt-proxy:8088/' + global_apt_http_proxy: 'https://some-global-apt-proxy:8088/' + ua_apt_http_proxy: 'http://10.0.10.10:3128' + ua_apt_https_proxy: 'https://10.0.10.10:3128' + enable: + - fips + """ + ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["ubuntu_advantage", "ubuntu-advantage"], } __doc__ = get_meta_doc(meta) @@ -78,7 +98,46 @@ LOG = logging.getLogger(__name__) -def configure_ua(token=None, enable=None): +def supplemental_schema_validation(ua_config): + """Validate user-provided ua:config option values. + + This function supplements flexible jsonschema validation with specific + value checks to aid in triage of invalid user-provided configuration. + + @param ua_config: Dictionary of config value under 'ubuntu_advantage'. + + @raises: ValueError describing invalid values provided. + """ + errors = [] + nl = "\n" + for key, value in sorted(ua_config.items()): + if key in ( + "http_proxy", + "https_proxy", + "global_apt_http_proxy", + "global_apt_https_proxy", + "ua_apt_http_proxy", + "ua_apt_https_proxy", + ): + try: + parsed_url = urlparse(value) + if parsed_url.scheme not in ("http", "https"): + errors.append( + f"Expected URL scheme http/https for ua:config:{key}." + f" Found: {value}" + ) + except (AttributeError, ValueError): + errors.append( + f"Expected a URL for ua:config:{key}. Found: {value}" + ) + + if errors: + raise ValueError( + f"Invalid ubuntu_advantage configuration:{nl}{nl.join(errors)}" + ) + + +def configure_ua(token=None, enable=None, config=None): """Call ua commandline client to attach or enable services.""" error = None if not token: @@ -102,6 +161,44 @@ def configure_ua(token=None, enable=None): ) enable = [] + if config is None: + config = dict() + elif not isinstance(config, dict): + LOG.warning( + "ubuntu_advantage: config should be a dict, not" + " a %s; skipping enabling config parameters", + type(config).__name__, + ) + config = dict() + + enable_errors = [] + + # UA Config + for key, value in sorted(config.items()): + if value is None: + LOG.debug("Unsetting UA config for %s", key) + config_cmd = ["ua", "config", "unset", key] + else: + LOG.debug("Setting UA config %s=%s", key, value) + if re.search(r"\s", value): + key_value = f"{key}={re.escape(value)}" + else: + key_value = f"{key}={value}" + config_cmd = ["ua", "config", "set", key_value] + + try: + subp.subp(config_cmd) + except subp.ProcessExecutionError as e: + enable_errors.append((key, e)) + + if enable_errors: + for param, error in enable_errors: + LOG.warning('Failure enabling "%s":\n%s', param, error) + raise RuntimeError( + "Failure enabling Ubuntu Advantage config(s): {}".format( + ", ".join('"{}"'.format(param) for param, _ in enable_errors) + ) + ) attach_cmd = ["ua", "attach", token] LOG.debug("Attaching to Ubuntu Advantage. %s", " ".join(attach_cmd)) try: @@ -176,9 +273,16 @@ def handle(name, cfg, cloud, log, args): LOG.error(msg) raise RuntimeError(msg) + config = ua_section.get("config") + + if config is not None: + supplemental_schema_validation(config) + maybe_install_ua_tools(cloud) configure_ua( - token=ua_section.get("token"), enable=ua_section.get("enable") + token=ua_section.get("token"), + enable=ua_section.get("enable"), + config=config, ) diff --git a/cloudinit/config/cc_ubuntu_autoinstall.py b/cloudinit/config/cc_ubuntu_autoinstall.py new file mode 100644 index 000000000..3d79c9ea0 --- /dev/null +++ b/cloudinit/config/cc_ubuntu_autoinstall.py @@ -0,0 +1,143 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Autoinstall: Support ubuntu live-server autoinstall syntax.""" + +import re +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit.config.schema import ( + MetaSchema, + SchemaProblem, + SchemaValidationError, + get_meta_doc, +) +from cloudinit.settings import PER_ONCE +from cloudinit.subp import subp + +LOG = logging.getLogger(__name__) + +distros = ["ubuntu"] + +meta: MetaSchema = { + "id": "cc_ubuntu_autoinstall", + "name": "Ubuntu Autoinstall", + "title": "Support Ubuntu live-server install syntax", + "description": dedent( + """\ + Ubuntu's autoinstall YAML supports single-system automated installs + in either the live-server install, via the ``subiquity`` snap, or the + next generation desktop installer, via `ubuntu-desktop-install` snap. + When "autoinstall" directives are provided in either + ``#cloud-config`` user-data or ``/etc/cloud/cloud.cfg.d`` validate + minimal autoinstall schema adherance and emit a warning if the + live-installer is not present. + + The live-installer will use autoinstall directives to seed answers to + configuration prompts during system install to allow for a + "touchless" or non-interactive Ubuntu system install. + + For more details on Ubuntu's autoinstaller: + https://ubuntu.com/server/docs/install/autoinstall + """ + ), + "distros": distros, + "examples": [ + dedent( + """\ + # Tell the live-server installer to provide dhcp6 network config + # and LVM on a disk matching the serial number prefix CT + autoinstall: + version: 1 + network: + version: 2 + ethernets: + enp0s31f6: + dhcp6: yes + storage: + layout: + name: lvm + match: + serial: CT* + """ + ) + ], + "frequency": PER_ONCE, + "activate_by_schema_keys": ["autoinstall"], +} + +__doc__ = get_meta_doc(meta) + + +LIVE_INSTALLER_SNAPS = ("subiquity", "ubuntu-desktop-installer") + + +def handle(name, cfg, cloud, log, _args): + + if "autoinstall" not in cfg: + LOG.debug( + "Skipping module named %s, no 'autoinstall' key in configuration", + name, + ) + return + + snap_list, _ = subp(["snap", "list"]) + installer_present = None + for snap_name in LIVE_INSTALLER_SNAPS: + if re.search(snap_name, snap_list): + installer_present = snap_name + if not installer_present: + LOG.warning( + "Skipping autoinstall module. Expected one of the Ubuntu" + " installer snap packages to be present: %s", + ", ".join(LIVE_INSTALLER_SNAPS), + ) + return + validate_config_schema(cfg) + LOG.debug( + "Valid autoinstall schema. Config will be processed by %s", + installer_present, + ) + + +def validate_config_schema(cfg): + """Supplemental runtime schema validation for autoinstall yaml. + + Schema validation issues currently result in a warning log currently which + can be easily ignored because warnings do not bubble up to cloud-init + status output. + + In the case of the live-installer, we want cloud-init to raise an error + to set overall cloud-init status to 'error' so it is more discoverable + in installer environments. + + # TODO(Drop this validation When cloud-init schema is strict and errors) + + :raise: SchemaValidationError if any known schema values are present. + """ + autoinstall_cfg = cfg["autoinstall"] + if not isinstance(autoinstall_cfg, dict): + raise SchemaValidationError( + [ + SchemaProblem( + "autoinstall", + "Expected dict type but found:" + f" {type(autoinstall_cfg).__name__}", + ) + ] + ) + + if "version" not in autoinstall_cfg: + raise SchemaValidationError( + [SchemaProblem("autoinstall", "Missing required 'version' key")] + ) + elif not isinstance(autoinstall_cfg.get("version"), int): + raise SchemaValidationError( + [ + SchemaProblem( + "autoinstall.version", + f"Expected int type but found:" + f" {type(autoinstall_cfg['version']).__name__}", + ) + ] + ) diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index 15f621a79..09e7badde 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -5,6 +5,14 @@ import os from textwrap import dedent +try: + import debconf + + HAS_DEBCONF = True +except ImportError: + debconf = None + HAS_DEBCONF = False + from cloudinit import log as logging from cloudinit import subp, temp_utils, type_utils, util from cloudinit.config.schema import MetaSchema, get_meta_doc @@ -34,6 +42,7 @@ ) ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["drivers"], } __doc__ = get_meta_doc(meta) @@ -48,10 +57,6 @@ # 'linux-restricted-modules' deb to accept the NVIDIA EULA and the package # will automatically link the drivers to the running kernel. -# EOL_XENIAL: can then drop this script and use python3-debconf which is only -# available in Bionic and later. Can't use python3-debconf currently as it -# isn't in Xenial and doesn't yet support X_LOADTEMPLATEFILE debconf command. - NVIDIA_DEBCONF_CONTENT = """\ Template: linux/nvidia/latelink Type: boolean @@ -61,13 +66,8 @@ make them available for use. """ -NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT = """\ -#!/bin/sh -# Allow cloud-init to trigger EULA acceptance via registering a debconf -# template to set linux/nvidia/latelink true -. /usr/share/debconf/confmodule -db_x_loadtemplatefile "$1" cloud-init -""" + +X_LOADTEMPLATEFILE = "X_LOADTEMPLATEFILE" def install_drivers(cfg, pkg_install_func): @@ -108,15 +108,10 @@ def install_drivers(cfg, pkg_install_func): # Register and set debconf selection linux/nvidia/latelink = true tdir = temp_utils.mkdtemp(needs_exe=True) debconf_file = os.path.join(tdir, "nvidia.template") - debconf_script = os.path.join(tdir, "nvidia-debconf.sh") try: util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT) - util.write_file( - debconf_script, - util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), - mode=0o755, - ) - subp.subp([debconf_script, debconf_file]) + with debconf.DebconfCommunicator("cloud-init") as dc: + dc.command(X_LOADTEMPLATEFILE, debconf_file) except Exception as e: util.logexc( LOG, "Failed to register NVIDIA debconf template: %s", str(e) @@ -143,5 +138,11 @@ def handle(name, cfg, cloud, log, _args): if "drivers" not in cfg: log.debug("Skipping module named %s, no 'drivers' key in config", name) return + if not HAS_DEBCONF: + log.warning( + "Skipping module named %s, 'python3-debconf' is not installed", + name, + ) + return install_drivers(cfg["drivers"], cloud.distro.install_packages) diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 5334f453f..56c52fe44 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -32,7 +32,7 @@ ping ``127.0.0.1`` or ``127.0.1.1`` or other ip). .. note:: - if ``manage_etc_hosts`` is set ``true`` or ``template``, the contents + if ``manage_etc_hosts`` is set ``true``, the contents of the hosts file will be updated every boot. To make any changes to the hosts file persistent they must be made in ``/etc/cloud/templates/hosts.tmpl`` @@ -88,6 +88,7 @@ ), ], "frequency": PER_ALWAYS, + "activate_by_schema_keys": ["manage_etc_hosts"], } __doc__ = get_meta_doc(meta) @@ -104,7 +105,7 @@ def handle(name, cfg, cloud, log, _args): "DEPRECATED: please use manage_etc_hosts: true instead of" " 'template'" ) - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn, _) = util.get_hostname_fqdn(cfg, cloud) if not hostname: log.warning( "Option 'manage_etc_hosts' was set, but no hostname was found" @@ -126,7 +127,7 @@ def handle(name, cfg, cloud, log, _args): ) elif manage_hosts == "localhost": - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn, _) = util.get_hostname_fqdn(cfg, cloud) if not hostname: log.warning( "Option 'manage_etc_hosts' was set, but no hostname was found" diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 1042abf3a..01d2078f7 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -73,6 +73,7 @@ ), ], "frequency": PER_ALWAYS, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) @@ -94,7 +95,12 @@ def handle(name, cfg, cloud, log, _args): if hostname_fqdn is not None: cloud.distro.set_option("prefer_fqdn_over_hostname", hostname_fqdn) - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn, is_default) = util.get_hostname_fqdn(cfg, cloud) + if is_default and hostname == "localhost": + # https://github.com/systemd/systemd/commit/d39079fcaa05e23540d2b1f0270fa31c22a7e9f1 + log.debug("Hostname is localhost. Let other services handle this.") + return + try: prev_fn = os.path.join(cloud.get_cpath("data"), "previous-hostname") log.debug("Updating hostname to %s (%s)", fqdn, hostname) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 96e63242c..612f172b5 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -141,11 +141,12 @@ ssh_import_id: [chad.smith] user: name: mynewdefault - sudo: false + sudo: null """ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": [], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/cc_wireguard.py b/cloudinit/config/cc_wireguard.py new file mode 100644 index 000000000..366aff400 --- /dev/null +++ b/cloudinit/config/cc_wireguard.py @@ -0,0 +1,295 @@ +# Author: Fabian Lichtenegger-Lukas +# Author: Josef Tschiggerl +# This file is part of cloud-init. See LICENSE file for license information. + +"""Wireguard""" +import re +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit import subp, util +from cloudinit.cloud import Cloud +from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.settings import PER_INSTANCE + +MODULE_DESCRIPTION = dedent( + """\ +Wireguard module provides a dynamic interface for configuring +Wireguard (as a peer or server) in an easy way. + +This module takes care of: + - writing interface configuration files + - enabling and starting interfaces + - installing wireguard-tools package + - loading wireguard kernel module + - executing readiness probes + +What's a readiness probe?\n +The idea behind readiness probes is to ensure Wireguard connectivity +before continuing the cloud-init process. This could be useful if you +need access to specific services like an internal APT Repository Server +(e.g Landscape) to install/update packages. + +Example:\n +An edge device can't access the internet but uses cloud-init modules which +will install packages (e.g landscape, packages, ubuntu_advantage). Those +modules will fail due to missing internet connection. The "wireguard" module +fixes that problem as it waits until all readinessprobes (which can be +arbitrary commands - e.g. checking if a proxy server is reachable over +Wireguard network) are finished before continuing the cloud-init +"config" stage. + +.. note:: + In order to use DNS with Wireguard you have to install ``resolvconf`` + package or symlink it to systemd's ``resolvectl``, otherwise ``wg-quick`` + commands will throw an error message that executable ``resolvconf`` is + missing which leads wireguard module to fail. +""" +) + +meta: MetaSchema = { + "id": "cc_wireguard", + "name": "Wireguard", + "title": "Module to configure Wireguard tunnel", + "description": MODULE_DESCRIPTION, + "distros": ["ubuntu"], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["wireguard"], + "examples": [ + dedent( + """\ + # Configure one or more WG interfaces and provide optional readinessprobes + wireguard: + interfaces: + - name: wg0 + config_path: /etc/wireguard/wg0.conf + content: | + [Interface] + PrivateKey = + Address =
+ [Peer] + PublicKey = + Endpoint = : + AllowedIPs = , , ... + - name: wg1 + config_path: /etc/wireguard/wg1.conf + content: | + [Interface] + PrivateKey = + Address =
+ [Peer] + PublicKey = + Endpoint = : + AllowedIPs = + readinessprobe: + - 'systemctl restart service' + - 'curl https://webhook.endpoint/example' + - 'nc -zv some-service-fqdn 443' + """ + ), + ], +} + +__doc__ = get_meta_doc(meta) + +LOG = logging.getLogger(__name__) + +REQUIRED_WG_INT_KEYS = frozenset(["name", "config_path", "content"]) +WG_CONFIG_FILE_MODE = 0o600 +NL = "\n" +MIN_KERNEL_VERSION = (5, 6) + + +def supplemental_schema_validation(wg_int: dict): + """Validate user-provided wg:interfaces option values. + + This function supplements flexible jsonschema validation with specific + value checks to aid in triage of invalid user-provided configuration. + + @param wg_int: Dict of configuration value under 'wg:interfaces'. + + @raises: ValueError describing invalid values provided. + """ + errors = [] + missing = REQUIRED_WG_INT_KEYS.difference(set(wg_int.keys())) + if missing: + keys = ", ".join(sorted(missing)) + errors.append(f"Missing required wg:interfaces keys: {keys}") + + for key, value in sorted(wg_int.items()): + if key == "name" or key == "config_path" or key == "content": + if not isinstance(value, str): + errors.append( + f"Expected a string for wg:interfaces:{key}. Found {value}" + ) + + if errors: + raise ValueError( + f"Invalid wireguard interface configuration:{NL}{NL.join(errors)}" + ) + + +def write_config(wg_int: dict): + """Writing user-provided configuration into Wireguard + interface configuration file. + + @param wg_int: Dict of configuration value under 'wg:interfaces'. + + @raises: RuntimeError for issues writing of configuration file. + """ + LOG.debug("Configuring Wireguard interface %s", wg_int["name"]) + try: + LOG.debug("Writing wireguard config to file %s", wg_int["config_path"]) + util.write_file( + wg_int["config_path"], wg_int["content"], mode=WG_CONFIG_FILE_MODE + ) + except Exception as e: + raise RuntimeError( + "Failure writing Wireguard configuration file" + f' {wg_int["config_path"]}:{NL}{str(e)}' + ) from e + + +def enable_wg(wg_int: dict, cloud: Cloud): + """Enable and start Wireguard interface + + @param wg_int: Dict of configuration value under 'wg:interfaces'. + + @raises: RuntimeError for issues enabling WG interface. + """ + try: + LOG.debug("Enabling wg-quick@%s at boot", wg_int["name"]) + cloud.distro.manage_service("enable", f'wg-quick@{wg_int["name"]}') + LOG.debug("Bringing up interface wg-quick@%s", wg_int["name"]) + cloud.distro.manage_service("start", f'wg-quick@{wg_int["name"]}') + except subp.ProcessExecutionError as e: + raise RuntimeError( + f"Failed enabling/starting Wireguard interface(s):{NL}{str(e)}" + ) from e + + +def readinessprobe_command_validation(wg_readinessprobes: list): + """Basic validation of user-provided probes + + @param wg_readinessprobes: List of readinessprobe probe(s). + + @raises: ValueError of wrong datatype provided for probes. + """ + errors = [] + pos = 0 + for c in wg_readinessprobes: + if not isinstance(c, str): + errors.append( + f"Expected a string for readinessprobe at {pos}. Found {c}" + ) + pos += 1 + + if errors: + raise ValueError( + f"Invalid readinessProbe commands:{NL}{NL.join(errors)}" + ) + + +def readinessprobe(wg_readinessprobes: list): + """Execute provided readiness probe(s) + + @param wg_readinessprobes: List of readinessprobe probe(s). + + @raises: ProcessExecutionError for issues during execution of probes. + """ + errors = [] + for c in wg_readinessprobes: + try: + LOG.debug("Running readinessprobe: '%s'", str(c)) + subp.subp(c, capture=True, shell=True) + except subp.ProcessExecutionError as e: + errors.append(f"{c}: {e}") + + if errors: + raise RuntimeError( + f"Failed running readinessprobe command:{NL}{NL.join(errors)}" + ) + + +def maybe_install_wireguard_packages(cloud: Cloud): + """Install wireguard packages and tools + + @param cloud: Cloud object + + @raises: Exception for issues during package + installation. + """ + + packages = ["wireguard-tools"] + + if subp.which("wg"): + return + + # Install DKMS when Kernel Verison lower 5.6 + if util.kernel_version() < MIN_KERNEL_VERSION: + packages.append("wireguard") + + try: + cloud.distro.update_package_sources() + except Exception: + util.logexc(LOG, "Package update failed") + raise + try: + cloud.distro.install_packages(packages) + except Exception: + util.logexc(LOG, "Failed to install wireguard-tools") + raise + + +def load_wireguard_kernel_module(): + """Load wireguard kernel module + + @raises: ProcessExecutionError for issues modprobe + """ + try: + out = subp.subp("lsmod", capture=True, shell=True) + if not re.search("wireguard", out.stdout.strip()): + LOG.debug("Loading wireguard kernel module") + subp.subp("modprobe wireguard", capture=True, shell=True) + except subp.ProcessExecutionError as e: + util.logexc(LOG, f"Could not load wireguard module:{NL}{str(e)}") + raise + + +def handle(name: str, cfg: dict, cloud: Cloud, log, args: list): + wg_section = None + + if "wireguard" in cfg: + LOG.debug("Found Wireguard section in config") + wg_section = cfg["wireguard"] + else: + LOG.debug( + "Skipping module named %s, no 'wireguard' configuration found", + name, + ) + return + + # install wireguard tools, enable kernel module + maybe_install_wireguard_packages(cloud) + load_wireguard_kernel_module() + + for wg_int in wg_section["interfaces"]: + # check schema + supplemental_schema_validation(wg_int) + + # write wg config files + write_config(wg_int) + + # enable wg interfaces + enable_wg(wg_int, cloud) + + # parse and run readinessprobe parameters + if ( + "readinessprobe" in wg_section + and wg_section["readinessprobe"] is not None + ): + wg_readinessprobes = wg_section["readinessprobe"] + readinessprobe_command_validation(wg_readinessprobes) + readinessprobe(wg_readinessprobes) + else: + LOG.debug("Skipping readinessprobe - no checks defined") diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 7cc7f854b..a020fac47 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -12,10 +12,10 @@ from cloudinit import log as logging from cloudinit import util +from cloudinit.cloud import Cloud from cloudinit.config.schema import MetaSchema, get_meta_doc from cloudinit.settings import PER_INSTANCE -DEFAULT_OWNER = "root:root" DEFAULT_PERMS = 0o644 DEFAULT_DEFER = False TEXT_PLAIN_ENC = "text/plain" @@ -108,12 +108,13 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["write_files"], } __doc__ = get_meta_doc(meta) -def handle(name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud: Cloud, log, _args): file_list = cfg.get("write_files", []) filtered_files = [ f @@ -127,7 +128,7 @@ def handle(name, cfg, _cloud, log, _args): name, ) return - write_files(name, filtered_files) + write_files(name, filtered_files, cloud.distro.default_owner) def canonicalize_extraction(encoding_type): @@ -155,7 +156,7 @@ def canonicalize_extraction(encoding_type): return [TEXT_PLAIN_ENC] -def write_files(name, files): +def write_files(name, files, owner: str): if not files: return @@ -171,7 +172,7 @@ def write_files(name, files): path = os.path.abspath(path) extractions = canonicalize_extraction(f_info.get("encoding")) contents = extract_contents(f_info.get("content", ""), extractions) - (u, g) = util.extract_usergroup(f_info.get("owner", DEFAULT_OWNER)) + (u, g) = util.extract_usergroup(f_info.get("owner", owner)) perms = decode_perms(f_info.get("permissions"), DEFAULT_PERMS) omode = "ab" if util.get_cfg_option_bool(f_info, "append") else "wb" util.write_file(path, contents, omode=omode, mode=perms) diff --git a/cloudinit/config/cc_write_files_deferred.py b/cloudinit/config/cc_write_files_deferred.py index dbbe90f6d..7cf5d5939 100644 --- a/cloudinit/config/cc_write_files_deferred.py +++ b/cloudinit/config/cc_write_files_deferred.py @@ -5,6 +5,7 @@ """Write Files Deferred: Defer writing certain files""" from cloudinit import util +from cloudinit.cloud import Cloud from cloudinit.config.cc_write_files import DEFAULT_DEFER, write_files from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS @@ -27,13 +28,14 @@ "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [], + "activate_by_schema_keys": ["write_files"], } # This module is undocumented in our schema docs __doc__ = "" -def handle(name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud: Cloud, log, _args): file_list = cfg.get("write_files", []) filtered_files = [ f @@ -47,4 +49,4 @@ def handle(name, cfg, _cloud, log, _args): name, ) return - write_files(name, filtered_files) + write_files(name, filtered_files, cloud.distro.default_owner) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index f73571926..0e683de20 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -29,6 +29,7 @@ "eurolinux", "fedora", "openEuler", + "openmandriva", "photon", "rhel", "rocky", @@ -99,8 +100,8 @@ # the repository file created. See: man yum.conf for supported # config keys. # - # Write /etc/yum.conf.d/my_package_stream.repo with gpgkey checks - # on the repo data of the repositoy enabled. + # Write /etc/yum.conf.d/my-package-stream.repo with gpgkey checks + # on the repo data of the repository enabled. yum_repos: my package stream: baseurl: http://blah.org/pub/epel/testing/5/$basearch/ @@ -112,15 +113,23 @@ ), ], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["yum_repos"], } __doc__ = get_meta_doc(meta) -def _canonicalize_id(repo_id): - repo_id = repo_id.lower().replace("-", "_") - repo_id = repo_id.replace(" ", "_") - return repo_id +def _canonicalize_id(repo_id: str) -> str: + """Canonicalize repo id. + + The sole name convention for repo ids is to not contain namespaces, + and typically the separator used is `-`. More info: + https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/sec-setting_repository_options + + :param repo_id: Repo id to convert. + :return: Canonical repo id. + """ + return repo_id.replace(" ", "-") def _format_repo_value(val): diff --git a/cloudinit/config/cc_zypper_add_repo.py b/cloudinit/config/cc_zypper_add_repo.py index 9b682bc64..02867b8f6 100644 --- a/cloudinit/config/cc_zypper_add_repo.py +++ b/cloudinit/config/cc_zypper_add_repo.py @@ -61,6 +61,7 @@ ) ], "frequency": PER_ALWAYS, + "activate_by_schema_keys": ["zypper"], } __doc__ = get_meta_doc(meta) diff --git a/cloudinit/config/modules.py b/cloudinit/config/modules.py index efb7a5a43..970343cd2 100644 --- a/cloudinit/config/modules.py +++ b/cloudinit/config/modules.py @@ -7,8 +7,8 @@ # This file is part of cloud-init. See LICENSE file for license information. import copy -from collections import namedtuple -from typing import List +from types import ModuleType +from typing import Dict, List, NamedTuple from cloudinit import config, importer from cloudinit import log as logging @@ -26,9 +26,13 @@ # we will not find something else with the same # name in the lookup path... MOD_PREFIX = "cc_" -ModuleDetails = namedtuple( - "ModuleDetails", ["module", "name", "frequency", "run_args"] -) + + +class ModuleDetails(NamedTuple): + module: ModuleType + name: str + frequency: str + run_args: List[str] def form_module_name(name): @@ -65,6 +69,17 @@ def validate_module(mod, name): ) +def _is_active(module_details: ModuleDetails, cfg: dict) -> bool: + activate_by_schema_keys_keys = frozenset( + module_details.module.meta.get("activate_by_schema_keys", {}) + ) + if not activate_by_schema_keys_keys: + return True + if not activate_by_schema_keys_keys.intersection(cfg.keys()): + return False + return True + + class Modules(object): def __init__(self, init: Init, cfg_files=None, reporter=None): self.init = init @@ -93,21 +108,21 @@ def cfg(self): # Only give out a copy so that others can't modify this... return copy.deepcopy(self._cached_cfg) - def _read_modules(self, name): + def _read_modules(self, name) -> List[Dict]: """Read the modules from the config file given the specified name. Returns a list of module definitions. E.g., [ { "mod": "bootcmd", - "freq": "always" + "freq": "always", "args": "some_arg", } ] Note that in the default case, only "mod" will be set. """ - module_list = [] + module_list: List[dict] = [] if name not in self.cfg: return module_list cfg_mods = self.cfg.get(name) @@ -155,7 +170,7 @@ def _read_modules(self, name): def _fixup_modules(self, raw_mods) -> List[ModuleDetails]: """Convert list of returned from _read_modules() into new format. - Invalid modules and arguments are ingnored. + Invalid modules and arguments are ignored. Also ensures that the module has the required meta fields. """ mostly_mods = [] @@ -269,12 +284,16 @@ def run_section(self, section_name): skipped = [] forced = [] overridden = self.cfg.get("unverified_modules", []) + inapplicable_mods = [] active_mods = [] - for (mod, name, _freq, _args) in mostly_mods: + for module_details in mostly_mods: + (mod, name, _freq, _args) = module_details if mod is None: continue worked_distros = mod.meta["distros"] - + if not _is_active(module_details, self.cfg): + inapplicable_mods.append(name) + continue # Skip only when the following conditions are all met: # - distros are defined in the module != ALL_DISTROS # - the current d_name isn't in distros @@ -288,6 +307,12 @@ def run_section(self, section_name): forced.append(name) active_mods.append([mod, name, _freq, _args]) + if inapplicable_mods: + LOG.info( + "Skipping modules '%s' because no applicable config " + "is provided.", + ",".join(inapplicable_mods), + ) if skipped: LOG.info( "Skipping modules '%s' because they are not verified " diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 4195a619c..427929855 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -7,16 +7,27 @@ import os import re import sys -import typing +import textwrap from collections import defaultdict +from collections.abc import Iterable from copy import deepcopy from functools import partial +from itertools import chain +from typing import TYPE_CHECKING, List, NamedTuple, Optional, Type, Union, cast import yaml from cloudinit import importer, safeyaml -from cloudinit.cmd.devel import read_cfg_paths -from cloudinit.util import error, find_modules, load_file +from cloudinit.stages import Init +from cloudinit.util import error, get_modules_from_dir, load_file + +try: + from jsonschema import ValidationError as _ValidationError + + ValidationError = _ValidationError +except ImportError: + ValidationError = Exception # type: ignore + error = partial(error, sys_exit=True) LOG = logging.getLogger(__name__) @@ -40,7 +51,7 @@ **Supported distros:** {distros} -{property_header} +{activate_by_schema_keys}{property_header} {property_doc} {examples} @@ -52,14 +63,18 @@ ) SCHEMA_EXAMPLES_HEADER = "**Examples**::\n\n" SCHEMA_EXAMPLES_SPACER_TEMPLATE = "\n # --- Example{0} ---" +DEPRECATED_KEY = "deprecated" +DEPRECATED_PREFIX = "DEPRECATED: " + +# type-annotate only if type-checking. +# Consider to add `type_extensions` as a dependency when Bionic is EOL. +if TYPE_CHECKING: + import typing -# annotations add value for development, but don't break old versions -# pyver: 3.6 -> 3.8 -# pylint: disable=E1101 -if sys.version_info >= (3, 8): + from typing_extensions import NotRequired, TypedDict - class MetaSchema(typing.TypedDict): + class MetaSchema(TypedDict): name: str id: str title: str @@ -67,30 +82,72 @@ class MetaSchema(typing.TypedDict): distros: typing.List[str] examples: typing.List[str] frequency: str + activate_by_schema_keys: NotRequired[List[str]] else: MetaSchema = dict -# pylint: enable=E1101 + + +class SchemaDeprecationError(ValidationError): + pass + + +class SchemaProblem(NamedTuple): + path: str + message: str + + def format(self) -> str: + return f"{self.path}: {self.message}" + + +SchemaProblems = List[SchemaProblem] + + +def _format_schema_problems( + schema_problems: SchemaProblems, + *, + prefix: Optional[str] = None, + separator: str = ", ", +) -> str: + formatted = separator.join(map(lambda p: p.format(), schema_problems)) + if prefix: + formatted = f"{prefix}{formatted}" + return formatted class SchemaValidationError(ValueError): """Raised when validating a cloud-config file against a schema.""" - def __init__(self, schema_errors=()): + def __init__( + self, + schema_errors: Optional[SchemaProblems] = None, + schema_deprecations: Optional[SchemaProblems] = None, + ): """Init the exception an n-tuple of schema errors. @param schema_errors: An n-tuple of the format: ((flat.config.key, msg),) + @param schema_deprecations: An n-tuple of the format: + ((flat.config.key, msg),) """ + message = "" + if schema_errors: + message += _format_schema_problems( + schema_errors, prefix="Cloud config schema errors: " + ) + if schema_deprecations: + if message: + message += "\n\n" + message += _format_schema_problems( + schema_deprecations, + prefix="Cloud config schema deprecations: ", + ) + super().__init__(message) self.schema_errors = schema_errors - error_messages = [ - "{0}: {1}".format(config_key, message) - for config_key, message in schema_errors - ] - message = "Cloud config schema errors: {0}".format( - ", ".join(error_messages) - ) - super(SchemaValidationError, self).__init__(message) + self.schema_deprecations = schema_deprecations + + def has_errors(self) -> bool: + return bool(self.schema_errors) def is_schema_byte_string(checker, instance): @@ -107,6 +164,113 @@ def is_schema_byte_string(checker, instance): ) or isinstance(instance, (bytes,)) +def _add_deprecation_msg(description: Optional[str] = None) -> str: + if description: + return f"{DEPRECATED_PREFIX}{description}" + return DEPRECATED_PREFIX.replace(":", ".").strip() + + +def _validator_deprecated( + _validator, + deprecated: bool, + _instance, + schema: dict, + error_type: Type[Exception] = SchemaDeprecationError, +): + """Jsonschema validator for `deprecated` items. + + It raises a instance of `error_type` if deprecated that must be handled, + otherwise the instance is consider faulty. + """ + if deprecated: + description = schema.get("description") + msg = _add_deprecation_msg(description) + yield error_type(msg) + + +def _anyOf( + validator, + anyOf, + instance, + _schema, + error_type: Type[Exception] = SchemaDeprecationError, +): + """Jsonschema validator for `anyOf`. + + It treats occurrences of `error_type` as non-errors, but yield them for + external processing. Useful to process schema annotations, as `deprecated`. + """ + from jsonschema import ValidationError + + all_errors = [] + all_deprecations = [] + for index, subschema in enumerate(anyOf): + all_errs = list( + validator.descend(instance, subschema, schema_path=index) + ) + errs = list(filter(lambda e: not isinstance(e, error_type), all_errs)) + deprecations = list( + filter(lambda e: isinstance(e, error_type), all_errs) + ) + if not errs: + all_deprecations.extend(deprecations) + break + all_errors.extend(errs) + else: + yield ValidationError( + "%r is not valid under any of the given schemas" % (instance,), + context=all_errors, + ) + yield from all_deprecations + + +def _oneOf( + validator, + oneOf, + instance, + _schema, + error_type: Type[Exception] = SchemaDeprecationError, +): + """Jsonschema validator for `oneOf`. + + It treats occurrences of `error_type` as non-errors, but yield them for + external processing. Useful to process schema annotations, as `deprecated`. + """ + from jsonschema import ValidationError + + subschemas = enumerate(oneOf) + all_errors = [] + all_deprecations = [] + for index, subschema in subschemas: + all_errs = list( + validator.descend(instance, subschema, schema_path=index) + ) + errs = list(filter(lambda e: not isinstance(e, error_type), all_errs)) + deprecations = list( + filter(lambda e: isinstance(e, error_type), all_errs) + ) + if not errs: + first_valid = subschema + all_deprecations.extend(deprecations) + break + all_errors.extend(errs) + else: + yield ValidationError( + "%r is not valid under any of the given schemas" % (instance,), + context=all_errors, + ) + + more_valid = [s for i, s in subschemas if validator.is_valid(instance, s)] + if more_valid: + more_valid.append(first_valid) + reprs = ", ".join(repr(schema) for schema in more_valid) + yield ValidationError( + "%r is valid under each of %s" % (instance, reprs) + ) + else: + yield from all_deprecations + + def get_jsonschema_validator(): """Get metaschema validator and format checker @@ -133,28 +297,51 @@ def get_jsonschema_validator(): # http://json-schema.org/understanding-json-schema/reference/object.html#pattern-properties strict_metaschema["properties"]["label"] = {"type": "string"} + validator_kwargs = {} if hasattr(Draft4Validator, "TYPE_CHECKER"): # jsonschema 3.0+ type_checker = Draft4Validator.TYPE_CHECKER.redefine( "string", is_schema_byte_string ) - cloudinitValidator = create( - meta_schema=strict_metaschema, - validators=Draft4Validator.VALIDATORS, - version="draft4", - type_checker=type_checker, - ) + validator_kwargs = { + "type_checker": type_checker, + } else: # jsonschema 2.6 workaround types = Draft4Validator.DEFAULT_TYPES # pylint: disable=E1101 # Allow bytes as well as string (and disable a spurious unsupported # assignment-operation pylint warning which appears because this # code path isn't written against the latest jsonschema). types["string"] = (str, bytes) # pylint: disable=E1137 - cloudinitValidator = create( # pylint: disable=E1123 - meta_schema=strict_metaschema, - validators=Draft4Validator.VALIDATORS, - version="draft4", - default_types=types, + validator_kwargs = {"default_types": types} + + # Add deprecation handling + validators = dict(Draft4Validator.VALIDATORS) + validators[DEPRECATED_KEY] = _validator_deprecated + validators["oneOf"] = _oneOf + validators["anyOf"] = _anyOf + + cloudinitValidator = create( + meta_schema=strict_metaschema, + validators=validators, + version="draft4", + **validator_kwargs, + ) + + # Add deprecation handling + def is_valid(self, instance, _schema=None, **__): + """Override version of `is_valid`. + + It does ignore instances of `SchemaDeprecationError`. + """ + errors = filter( + lambda e: not isinstance( # pylint: disable=W1116 + e, SchemaDeprecationError + ), + self.iter_errors(instance, _schema), ) + return next(errors, None) is None + + cloudinitValidator.is_valid = is_valid + return (cloudinitValidator, FormatChecker) @@ -180,9 +367,11 @@ def validate_cloudconfig_metaschema(validator, schema: dict, throw=True): # sites if throw: raise SchemaValidationError( - schema_errors=( - (".".join([str(p) for p in err.path]), err.message), - ) + schema_errors=[ + SchemaProblem( + ".".join([str(p) for p in err.path]), err.message + ) + ] ) from err LOG.warning( "Meta-schema validation failed, attempting to validate config " @@ -197,6 +386,7 @@ def validate_cloudconfig_schema( strict: bool = False, strict_metaschema: bool = False, log_details: bool = True, + log_deprecations: bool = False, ): """Validate provided config meets the schema definition. @@ -212,6 +402,7 @@ def validate_cloudconfig_schema( @param log_details: Boolean, when True logs details of validation errors. If there are concerns about logging sensitive userdata, this should be set to False. + @param log_deprecations: Controls whether to log deprecations or not. @raises: SchemaValidationError when provided config does not validate against the provided schema. @@ -230,76 +421,181 @@ def validate_cloudconfig_schema( return validator = cloudinitValidator(schema, format_checker=FormatChecker()) - errors = () + + errors: SchemaProblems = [] + deprecations: SchemaProblems = [] for error in sorted(validator.iter_errors(config), key=lambda e: e.path): path = ".".join([str(p) for p in error.path]) - errors += ((path, error.message),) + problem = (SchemaProblem(path, error.message),) + if isinstance(error, SchemaDeprecationError): # pylint: disable=W1116 + deprecations += problem + else: + errors += problem + + if log_deprecations and deprecations: + message = _format_schema_problems( + deprecations, + prefix="Deprecated cloud-config provided:\n", + separator="\n", + ) + LOG.warning(message) + if strict and (errors or deprecations): + raise SchemaValidationError(errors, deprecations) if errors: - if strict: - # This could output/log sensitive data - raise SchemaValidationError(errors) if log_details: - messages = ["{0}: {1}".format(k, msg) for k, msg in errors] - details = "\n" + "\n".join(messages) + details = _format_schema_problems( + errors, + prefix="Invalid cloud-config provided:\n", + separator="\n", + ) else: details = ( + "Invalid cloud-config provided: " "Please run 'sudo cloud-init schema --system' to " "see the schema errors." ) - LOG.warning("Invalid cloud-config provided: %s", details) + LOG.warning(details) + + +class _Annotator: + def __init__( + self, + cloudconfig: dict, + original_content: bytes, + schemamarks: dict, + ): + self._cloudconfig = cloudconfig + self._original_content = original_content + self._schemamarks = schemamarks + + @staticmethod + def _build_footer(title: str, content: List[str]) -> str: + body = "\n".join(content) + return f"# {title}: -------------\n{body}\n\n" + + def _build_errors_by_line(self, schema_problems: SchemaProblems): + errors_by_line = defaultdict(list) + for (path, msg) in schema_problems: + match = re.match(r"format-l(?P\d+)\.c(?P\d+).*", path) + if match: + line, col = match.groups() + errors_by_line[int(line)].append(msg) + else: + col = None + errors_by_line[self._schemamarks[path]].append(msg) + if col is not None: + msg = "Line {line} column {col}: {msg}".format( + line=line, col=col, msg=msg + ) + return errors_by_line + + @staticmethod + def _add_problems( + problems: List[str], + labels: List[str], + footer: List[str], + index: int, + label_prefix: str = "", + ) -> int: + for problem in problems: + label = f"{label_prefix}{index}" + labels.append(label) + footer.append(f"# {label}: {problem}") + index += 1 + return index + + def _annotate_content( + self, + lines: List[str], + errors_by_line: dict, + deprecations_by_line: dict, + ) -> List[str]: + annotated_content = [] + error_footer: List[str] = [] + deprecation_footer: List[str] = [] + error_index = 1 + deprecation_index = 1 + for line_number, line in enumerate(lines, 1): + errors = errors_by_line[line_number] + deprecations = deprecations_by_line[line_number] + if errors or deprecations: + labels: List[str] = [] + error_index = self._add_problems( + errors, labels, error_footer, error_index, label_prefix="E" + ) + deprecation_index = self._add_problems( + deprecations, + labels, + deprecation_footer, + deprecation_index, + label_prefix="D", + ) + annotated_content.append(line + "\t\t# " + ",".join(labels)) + else: + annotated_content.append(line) + + annotated_content.extend( + map( + lambda seq: self._build_footer(*seq), + filter( + lambda seq: bool(seq[1]), + ( + ("Errors", error_footer), + ("Deprecations", deprecation_footer), + ), + ), + ) + ) + return annotated_content + + def annotate( + self, + schema_errors: SchemaProblems, + schema_deprecations: SchemaProblems, + ) -> Union[str, bytes]: + if not schema_errors and not schema_deprecations: + return self._original_content + lines = self._original_content.decode().split("\n") + if not isinstance(self._cloudconfig, dict): + # Return a meaningful message on empty cloud-config + return "\n".join( + lines + + [ + self._build_footer( + "Errors", ["# E1: Cloud-config is not a YAML dict."] + ) + ] + ) + errors_by_line = self._build_errors_by_line(schema_errors) + deprecations_by_line = self._build_errors_by_line(schema_deprecations) + annotated_content = self._annotate_content( + lines, errors_by_line, deprecations_by_line + ) + return "\n".join(annotated_content) def annotated_cloudconfig_file( - cloudconfig, original_content, schema_errors, schemamarks -): + cloudconfig: dict, + original_content: bytes, + schemamarks: dict, + *, + schema_errors: Optional[SchemaProblems] = None, + schema_deprecations: Optional[SchemaProblems] = None, +) -> Union[str, bytes]: """Return contents of the cloud-config file annotated with schema errors. @param cloudconfig: YAML-loaded dict from the original_content or empty dict if unparseable. @param original_content: The contents of a cloud-config file - @param schema_errors: List of tuples from a JSONSchemaValidationError. The - tuples consist of (schemapath, error_message). - """ - if not schema_errors: - return original_content - errors_by_line = defaultdict(list) - error_footer = [] - error_header = "# Errors: -------------\n{0}\n\n" - annotated_content = [] - lines = original_content.decode().split("\n") - if not isinstance(cloudconfig, dict): - # Return a meaningful message on empty cloud-config - return "\n".join( - lines - + [error_header.format("# E1: Cloud-config is not a YAML dict.")] - ) - for path, msg in schema_errors: - match = re.match(r"format-l(?P\d+)\.c(?P\d+).*", path) - if match: - line, col = match.groups() - errors_by_line[int(line)].append(msg) - else: - col = None - errors_by_line[schemamarks[path]].append(msg) - if col is not None: - msg = "Line {line} column {col}: {msg}".format( - line=line, col=col, msg=msg - ) - error_index = 1 - for line_number, line in enumerate(lines, 1): - errors = errors_by_line[line_number] - if errors: - error_label = [] - for error in errors: - error_label.append("E{0}".format(error_index)) - error_footer.append("# E{0}: {1}".format(error_index, error)) - error_index += 1 - annotated_content.append(line + "\t\t# " + ",".join(error_label)) + @param schemamarks: Dict with schema marks. + @param schema_errors: Instance of `SchemaProblems`. + @param schema_deprecations: Instance of `SchemaProblems`. - else: - annotated_content.append(line) - annotated_content.append(error_header.format("\n".join(error_footer))) - return "\n".join(annotated_content) + @return Annotated schema + """ + return _Annotator(cloudconfig, original_content, schemamarks).annotate( + schema_errors or [], schema_deprecations or [] + ) def validate_cloudconfig_file(config_path, schema, annotate=False): @@ -321,9 +617,10 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): "Unable to read system userdata as non-root user." " Try using sudo" ) - paths = read_cfg_paths() - user_data_file = paths.get_ipath_cur("userdata_raw") - content = load_file(user_data_file, decode=False) + init = Init(ds_deps=[]) + init.fetch(existing="trust") + init.consume_data() + content = load_file(init.paths.get_ipath("cloud_config"), decode=False) else: if not os.path.exists(config_path): raise RuntimeError( @@ -331,19 +628,19 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): ) content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): - errors = ( - ( + errors = [ + SchemaProblem( "format-l1.c1", 'File {0} needs to begin with "{1}"'.format( config_path, CLOUD_CONFIG_HEADER.decode() ), ), - ) + ] error = SchemaValidationError(errors) if annotate: print( annotated_cloudconfig_file( - {}, content, error.schema_errors, {} + {}, content, {}, schema_errors=error.schema_errors ) ) raise error @@ -363,17 +660,17 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): if mark: line = mark.line + 1 column = mark.column + 1 - errors = ( - ( + errors = [ + SchemaProblem( "format-l{line}.c{col}".format(line=line, col=column), "File {0} is not valid yaml. {1}".format(config_path, str(e)), ), - ) + ] error = SchemaValidationError(errors) if annotate: print( annotated_cloudconfig_file( - {}, content, error.schema_errors, {} + {}, content, {}, schema_errors=error.schema_errors ) ) raise error from e @@ -382,15 +679,29 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): if not annotate: raise RuntimeError("Cloud-config is not a YAML dict.") try: - validate_cloudconfig_schema(cloudconfig, schema, strict=True) + validate_cloudconfig_schema( + cloudconfig, schema, strict=True, log_deprecations=False + ) except SchemaValidationError as e: if annotate: print( annotated_cloudconfig_file( - cloudconfig, content, e.schema_errors, marks + cloudconfig, + content, + marks, + schema_errors=e.schema_errors, + schema_deprecations=e.schema_deprecations, ) ) - raise + else: + message = _format_schema_problems( + e.schema_deprecations, + prefix="Cloud config schema deprecations: ", + separator=", ", + ) + print(message) + if e.has_errors(): # We do not consider deprecations as error + raise def _sort_property_order(value): @@ -406,6 +717,31 @@ def _sort_property_order(value): return 0 +def _flatten(xs): + for x in xs: + if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): + yield from _flatten(x) + else: + yield x + + +def _collect_subschema_types(property_dict: dict, multi_key: str) -> List[str]: + property_types = [] + for subschema in property_dict.get(multi_key, {}): + if subschema.get(DEPRECATED_KEY): # don't document deprecated types + continue + if subschema.get("enum"): + property_types.extend( + [ + f"``{_YAML_MAP.get(enum_value, enum_value)}``" + for enum_value in subschema.get("enum", []) + ] + ) + elif subschema.get("type"): + property_types.append(subschema["type"]) + return list(_flatten(property_types)) + + def _get_property_type(property_dict: dict, defs: dict) -> str: """Return a string representing a property type from a given jsonschema. @@ -414,18 +750,15 @@ def _get_property_type(property_dict: dict, defs: dict) -> str: property_types = property_dict.get("type", []) if not isinstance(property_types, list): property_types = [property_types] + # A property_dict cannot have simultaneously more than one of these props if property_dict.get("enum"): property_types = [ f"``{_YAML_MAP.get(k, k)}``" for k in property_dict["enum"] ] elif property_dict.get("oneOf"): - property_types.extend( - [ - subschema["type"] - for subschema in property_dict.get("oneOf") - if subschema.get("type") - ] - ) + property_types.extend(_collect_subschema_types(property_dict, "oneOf")) + elif property_dict.get("anyOf"): + property_types.extend(_collect_subschema_types(property_dict, "anyOf")) if len(property_types) == 1: property_type = property_types[0] else: @@ -436,8 +769,14 @@ def _get_property_type(property_dict: dict, defs: dict) -> str: if not isinstance(sub_property_types, list): sub_property_types = [sub_property_types] # Collect each item type - for sub_item in items.get("oneOf", {}): - sub_property_types.append(_get_property_type(sub_item, defs)) + prune_undefined = bool(sub_property_types) + for sub_item in chain(items.get("oneOf", {}), items.get("anyOf", {})): + sub_type = _get_property_type(sub_item, defs) + if prune_undefined and sub_type == "UNDEFINED": + # If the main object has a type, then sub-schemas are allowed to + # omit the type. Prune subschema undefined types. + continue + sub_property_types.append(sub_type) if sub_property_types: if len(sub_property_types) == 1: return f"{property_type} of {sub_property_types[0]}" @@ -481,14 +820,70 @@ def _flatten_schema_refs(src_cfg: dict, defs: dict): # Update the references in subschema for doc rendering src_cfg["items"].update(defs[reference]) if "oneOf" in src_cfg["items"]: - for alt_schema in src_cfg["items"]["oneOf"]: - if "$ref" in alt_schema: - reference = alt_schema.pop("$ref").replace("#/$defs/", "") - alt_schema.update(defs[reference]) - for alt_schema in src_cfg.get("oneOf", []): - if "$ref" in alt_schema: - reference = alt_schema.pop("$ref").replace("#/$defs/", "") - alt_schema.update(defs[reference]) + for sub_schema in src_cfg["items"]["oneOf"]: + if "$ref" in sub_schema: + reference = sub_schema.pop("$ref").replace("#/$defs/", "") + sub_schema.update(defs[reference]) + for sub_schema in chain( + src_cfg.get("oneOf", []), + src_cfg.get("anyOf", []), + src_cfg.get("allOf", []), + ): + if "$ref" in sub_schema: + reference = sub_schema.pop("$ref").replace("#/$defs/", "") + sub_schema.update(defs[reference]) + + +def _flatten_schema_all_of(src_cfg: dict): + """Flatten schema: Merge allOf. + + If a schema as allOf, then all of the sub-schemas must hold. Therefore + it is safe to merge them. + """ + sub_schemas = src_cfg.pop("allOf", None) + if not sub_schemas: + return + for sub_schema in sub_schemas: + src_cfg.update(sub_schema) + + +def _get_property_description(prop_config: dict) -> str: + """Return accumulated property description. + + Account for the following keys: + - top-level description key + - any description key present in each subitem under anyOf or allOf + + Order and deprecated property description after active descriptions. + Add a trailing stop "." to any description not ending with ":". + """ + prop_descr = prop_config.get("description", "") + oneOf = prop_config.get("oneOf", {}) + anyOf = prop_config.get("anyOf", {}) + descriptions = [] + deprecated_descriptions = [] + if prop_descr: + prop_descr = prop_descr.rstrip(".") + if not prop_config.get(DEPRECATED_KEY): + descriptions.append(prop_descr) + else: + deprecated_descriptions.append(_add_deprecation_msg(prop_descr)) + for sub_item in chain(oneOf, anyOf): + if not sub_item.get("description"): + continue + if not sub_item.get(DEPRECATED_KEY): + descriptions.append(sub_item["description"].rstrip(".")) + else: + deprecated_descriptions.append( + f"{DEPRECATED_PREFIX}{sub_item['description'].rstrip('.')}" + ) + # order deprecated descrs last + description = ". ".join(chain(descriptions, deprecated_descriptions)) + if description: + description = f" {description}" + if description[-1] != ":": + description += "." + return description def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: @@ -507,12 +902,11 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: for prop_schema in property_schemas: for prop_key, prop_config in prop_schema.items(): _flatten_schema_refs(prop_config, defs) + _flatten_schema_all_of(prop_config) if prop_config.get("hidden") is True: continue # document nothing for this property - # Define prop_name and description for SCHEMA_PROPERTY_TMPL - description = prop_config.get("description", "") - if description: - description = " " + description + + description = _get_property_description(prop_config) # Define prop_name and description for SCHEMA_PROPERTY_TMPL label = prop_config.get("label", prop_key) @@ -571,9 +965,7 @@ def _get_examples(meta: MetaSchema) -> str: return "" rst_content = SCHEMA_EXAMPLES_HEADER for count, example in enumerate(examples): - # Python2.6 is missing textwrapper.indent - lines = example.split("\n") - indented_lines = [" {0}".format(line) for line in lines] + indented_lines = textwrap.indent(example, " ").split("\n") if rst_content != SCHEMA_EXAMPLES_HEADER: indented_lines.insert( 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1) @@ -582,7 +974,16 @@ def _get_examples(meta: MetaSchema) -> str: return rst_content -def get_meta_doc(meta: MetaSchema, schema: dict = None) -> str: +def _get_activate_by_schema_keys_doc(meta: MetaSchema) -> str: + if not meta.get("activate_by_schema_keys"): + return "" + schema_keys = ", ".join( + f"``{k}``" for k in meta["activate_by_schema_keys"] + ) + return f"**Activate only on keys:** {schema_keys}\n\n" + + +def get_meta_doc(meta: MetaSchema, schema: Optional[dict] = None) -> str: """Return reStructured text rendering the provided metadata. @param meta: Dict of metadata to render. @@ -595,26 +996,25 @@ def get_meta_doc(meta: MetaSchema, schema: dict = None) -> str: if not meta or not schema: raise ValueError("Expected non-empty meta and schema") keys = set(meta.keys()) - expected = set( - { - "id", - "title", - "examples", - "frequency", - "distros", - "description", - "name", - } - ) + required_keys = { + "id", + "title", + "examples", + "frequency", + "distros", + "description", + "name", + } + optional_keys = {"activate_by_schema_keys"} error_message = "" - if expected - keys: - error_message = "Missing expected keys in module meta: {}".format( - expected - keys + if required_keys - keys: + error_message = "Missing required keys in module meta: {}".format( + required_keys - keys ) - elif keys - expected: + elif keys - required_keys - optional_keys: error_message = ( "Additional unexpected keys found in module meta: {}".format( - keys - expected + keys - required_keys ) ) if error_message: @@ -625,7 +1025,8 @@ def get_meta_doc(meta: MetaSchema, schema: dict = None) -> str: meta_copy["property_header"] = "" defs = schema.get("$defs", {}) if defs.get(meta["id"]): - schema = defs.get(meta["id"]) + schema = defs.get(meta["id"], {}) + schema = cast(dict, schema) try: meta_copy["property_doc"] = _get_property_doc(schema, defs=defs) except AttributeError: @@ -637,13 +1038,16 @@ def get_meta_doc(meta: MetaSchema, schema: dict = None) -> str: meta_copy["distros"] = ", ".join(meta["distros"]) # Need an underbar of the same length as the name meta_copy["title_underbar"] = re.sub(r".", "-", meta["name"]) + meta_copy["activate_by_schema_keys"] = _get_activate_by_schema_keys_doc( + meta + ) template = SCHEMA_DOC_TMPL.format(**meta_copy) return template def get_modules() -> dict: configs_dir = os.path.dirname(os.path.abspath(__file__)) - return find_modules(configs_dir) + return get_modules_from_dir(configs_dir) def load_doc(requested_modules: list) -> str: diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index d409d5d67..b7124cb74 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "$defs": { "users_groups.groups_by_groupname": { + "additionalProperties": false, "patternProperties": { "^.+$": { "label": "", @@ -19,6 +20,7 @@ {"required": ["name"]}, {"required": ["snapuser"]} ], + "additionalProperties": false, "properties": { "name": { "description": "The user's login name. Required otherwise user creation will be skipped for this user.", @@ -27,7 +29,8 @@ "expiredate": { "default": null, "description": "Optional. Date on which the user's account will be disabled. Default: ``null``", - "type": "string" + "type": "string", + "format": "date" }, "gecos": { "description": "Optional comment about the user, usually a comma-separated string of real name and contact information", @@ -35,7 +38,29 @@ }, "groups": { "description": "Optional comma-separated string of groups to add the user to.", - "type": "string" + "oneOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "type": ["string"] + }, + "minItems": 1 + }, + { + "type": "object", + "patternProperties": { + "^.+$": { + "label": "", + "description": "When providing an object for users.groups the ```` keys are the groups to add this user to", + "deprecated": true, + "type": ["null"], + "minItems": 1 + } + }, + "hidden": ["patternProperties"] + } + ] }, "homedir": { "description": "Optional home dir for user. Default: ``/home/``", @@ -46,6 +71,12 @@ "description": "Optional string representing the number of days until the user is disabled. ", "type": "string" }, + "lock-passwd": { + "default": true, + "description": "Dropped after April 2027. Use ``lock_passwd``. Default: ``true``", + "type": "boolean", + "deprecated": true + }, "lock_passwd": { "default": true, "description": "Disable password login. Default: ``true``", @@ -67,15 +98,15 @@ "type": "boolean" }, "passwd": { - "description": "Hash of user password applied when user does not exist. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While hashed password is better than plain text, using ``passwd`` in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", + "description": "Hash of user password applied when user does not exist. This will NOT be applied if the user already exists. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While hashed password is better than plain text, using ``passwd`` in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", "type": "string" }, "hashed_passwd": { - "description": "Hash of user password applied to new or existing users. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While ``hashed_password`` is better than ``plain_text_passwd``, using ``passwd`` in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", + "description": "Hash of user password to be applied. This will be applied even if the user is pre-existing. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While ``hashed_password`` is better than ``plain_text_passwd``, using ``passwd`` in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", "type": "string" }, "plain_text_passwd": { - "description": "Clear text of user password applied to new or existing users. There are many more secure options than using plain text passwords, such as ``ssh_import_id`` or ``hashed_passwd``. Do not use this in production as user-data and your password can be exposed.", + "description": "Clear text of user password to be applied. This will be applied even if the user is pre-existing. There are many more secure options than using plain text passwords, such as ``ssh_import_id`` or ``hashed_passwd``. Do not use this in production as user-data and your password can be exposed.", "type": "string" }, "create_groups": { @@ -123,22 +154,37 @@ "default": false }, "sudo": { - "type": ["boolean", "string"], - "description": "Sudo rule to use or false. Absence of a sudo value or ``false`` will result in no sudo rules added for this user. DEPRECATED: the value ``false`` will be deprecated in the future release. Use ``null`` or no ``sudo`` key instead." + "oneOf": [ + { + "type": ["string", "null"], + "description": "Sudo rule to use or false. Absence of a sudo value or ``null`` will result in no sudo rules added for this user." + }, + { + "type": "boolean", + "deprecated": true, + "description": "The value ``false`` will be dropped after April 2027. Use ``null`` or no ``sudo`` key instead." + } + ] }, "uid": { "description": "The user's ID. Default is next available value.", - "type": "integer" + "oneOf": [ + {"type": "integer"}, + { + "type": "string", + "description": "The use of ``string`` type will be dropped after April 2027. Use an ``integer`` instead.", + "deprecated": true + } + ] } - }, - "additionalProperties": false + } }, "apt_configure.mirror": { "type": "array", "items": { "type": "object", - "additionalProperties": false, "required": ["arches"], + "additionalProperties": false, "properties": { "arches": { "type": "array", @@ -163,12 +209,13 @@ }, "ca_certs.properties": { "type": "object", + "additionalProperties": false, "properties": { "remove-defaults": { - "description": "DEPRECATED. Use ``remove_defaults``. ", - "deprecated": true, + "description": "Dropped after April 2027. Use ``remove_defaults``.", "type": "boolean", - "default": false + "default": false, + "deprecated": true }, "remove_defaults": { "description": "Remove default CA certificates if true. Default: false", @@ -182,14 +229,126 @@ "minItems": 1 } }, - "additionalProperties": false, "minProperties": 1 }, + "cc_ubuntu_autoinstall": { + "type": "object", + "properties": { + "autoinstall": { + "description": "Opaque autoinstall schema definition for Ubuntu autoinstall. Full schema processed by live-installer. See: https://ubuntu.com/server/docs/install/autoinstall-reference", + "type": "object", + "properties": { + "version": { + "type": "integer" + } + }, + "required": ["version"] + } + }, + "additionalProperties": true + }, + "cc_ansible": { + "type": "object", + "properties": { + "ansible": { + "type": "object", + "additionalProperties": false, + "properties": { + "install-method": { + "type": "string", + "default": "distro", + "enum": [ + "distro", + "pip" + ], + "description": "The type of installation for ansible. It can be one of the following values:\n\n - ``distro``\n - ``pip``" + }, + "package-name": { + "type": "string", + "default": "ansible" + }, + "pull": { + "required": ["url", "playbook-name"], + "type": "object", + "additionalProperties": false, + "properties": { + "accept-host-key": { + "type": "boolean", + "default": false + }, + "clean": { + "type": "boolean", + "default": false + }, + "full": { + "type": "boolean", + "default": false + }, + "diff": { + "type": "boolean", + "default": false + }, + "ssh-common-args": { + "type": "string" + }, + "scp-extra-args": { + "type": "string" + }, + "sftp-extra-args": { + "type": "string" + }, + "private-key": { + "type": "string" + }, + "checkout": { + "type": "string" + }, + "module-path": { + "type": "string" + }, + "timeout": { + "type": "string" + }, + "url": { + "type": "string" + }, + "connection": { + "type": "string" + }, + "vault-id": { + "type": "string" + }, + "vault-password-file": { + "type": "string" + }, + "module-name": { + "type": "string" + }, + "sleep": { + "type": "string" + }, + "tags": { + "type": "string" + }, + "skip-tags": { + "type": "string" + }, + "playbook-name": { + "type": "string" + } + } + } + } + } + } + }, "cc_apk_configure": { "type": "object", "properties": { "apk_repos": { "type": "object", + "minProperties": 1, + "additionalProperties": false, "properties": { "preserve_repositories": { "type": "boolean", @@ -198,6 +357,7 @@ }, "alpine_repo": { "type": ["object", "null"], + "additionalProperties": false, "properties": { "base_url": { "type": "string", @@ -220,16 +380,13 @@ } }, "required": ["version"], - "minProperties": 1, - "additionalProperties": false + "minProperties": 1 }, "local_repo_base_url": { "type": "string", "description": "The base URL of an Alpine repository containing unofficial packages" } - }, - "minProperties": 1, - "additionalProperties": false + } } } }, @@ -237,8 +394,8 @@ "properties": { "apt": { "type": "object", - "additionalProperties": false, "minProperties": 1, + "additionalProperties": false, "properties": { "preserve_sources_list": { "type": "boolean", @@ -268,6 +425,7 @@ "debconf_selections": { "type": "object", "minProperties": 1, + "additionalProperties": false, "patternProperties": { "^.+$": { "type": "string" @@ -301,9 +459,11 @@ }, "sources": { "type": "object", + "additionalProperties": false, "patternProperties": { "^.+$": { "type": "object", + "additionalProperties": false, "properties": { "source": { "type": "string" @@ -321,7 +481,6 @@ "type": "string" } }, - "additionalProperties": false, "minProperties": 1 } }, @@ -384,7 +543,13 @@ "$ref": "#/$defs/ca_certs.properties" }, "ca-certs": { - "$ref": "#/$defs/ca_certs.properties" + "allOf": [ + {"$ref": "#/$defs/ca_certs.properties"}, + { + "deprecated": true, + "description": "Dropped after April 2027. Use ``ca_certs``." + } + ] } } }, @@ -393,8 +558,8 @@ "properties": { "chef": { "type": "object", - "additionalProperties": false, "minProperties": 1, + "additionalProperties": false, "properties": { "directories": { "type": "array", @@ -536,26 +701,6 @@ } } }, - "cc_debug": { - "type": "object", - "properties": { - "debug": { - "additionalProperties": false, - "minProperties": 1, - "type": "object", - "properties": { - "verbose": { - "description": "Should always be true for this module", - "type": "boolean" - }, - "output": { - "description": "Location to write output. Defaults to console + log", - "type": "string" - } - } - } - } - }, "cc_disable_ec2_metadata": { "type": "object", "properties": { @@ -571,6 +716,7 @@ "properties": { "device_aliases": { "type": "object", + "additionalProperties": false, "patternProperties": { "^.+$": { "label": "", @@ -581,6 +727,7 @@ }, "disk_setup": { "type": "object", + "additionalProperties": false, "patternProperties": { "^.+$": { "label": "", @@ -594,7 +741,6 @@ "description": "Specifies the partition table type, either ``mbr`` or ``gpt``. Default: ``mbr``." }, "layout": { - "type": ["string", "boolean", "array"], "default": false, "oneOf": [ {"type": "string", "enum": ["remove"]}, @@ -681,8 +827,8 @@ "properties": { "fan": { "type": "object", - "additionalProperties": false, "required": ["config"], + "additionalProperties": false, "properties": { "config": { "type": "string", @@ -714,9 +860,18 @@ "additionalProperties": false, "properties": { "mode": { - "enum": [false, "auto", "growpart", "gpart", "off"], "default": "auto", - "description": "The utility to use for resizing. Default: ``auto``\n\nPossible options:\n\n* ``auto`` - Use any available utility\n\n* ``growpart`` - Use growpart utility\n\n* ``gpart`` - Use BSD gpart utility\n\n* ``off`` - Take no action\n\nSpecifying a boolean ``false`` value for this key is deprecated. Use ``off`` instead." + "description": "The utility to use for resizing. Default: ``auto``\n\nPossible options:\n\n* ``auto`` - Use any available utility\n\n* ``growpart`` - Use growpart utility\n\n* ``gpart`` - Use BSD gpart utility\n\n* ``off`` - Take no action.", + "oneOf": [ + { + "enum": ["auto", "growpart", "gpart", "off"] + }, + { + "enum": [false], + "description": "Specifying a boolean ``false`` value for this key is deprecated. Use ``off`` instead.", + "deprecated": true + } + ] }, "devices": { "type": "array", @@ -752,14 +907,24 @@ "description": "Device to use as target for grub installation. If unspecified, ``grub-probe`` of ``/boot`` will be used to find the device" }, "grub-pc/install_devices_empty": { - "type": ["string", "boolean"], - "description": "Sets values for ``grub-pc/install_devices_empty``. If unspecified, will be set to ``true`` if ``grub-pc/install_devices`` is empty, otherwise ``false``. Using a non-boolean value for this field is deprecated." + "description": "Sets values for ``grub-pc/install_devices_empty``. If unspecified, will be set to ``true`` if ``grub-pc/install_devices`` is empty, otherwise ``false``.", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "description": "Use a boolean value instead.", + "deprecated": true + } + ] } } }, "grub-dpkg": { "type": "object", - "description": "DEPRECATED: Use ``grub_dpkg`` instead" + "description": "Use ``grub_dpkg`` instead", + "deprecated": true } } }, @@ -777,10 +942,8 @@ "properties": { "when": { "type": "array", - "additionalProperties": false, "items": { "type": "string", - "additionalProperties": false, "enum": [ "boot-new-instance", "boot-legacy", @@ -800,6 +963,7 @@ "properties": { "keyboard": { "type": "object", + "additionalProperties": false, "properties": { "layout": { "type": "string", @@ -819,8 +983,7 @@ "description": "Optional. Keyboard options. Corresponds to XKBOPTIONS." } }, - "required": ["layout"], - "additionalProperties": false + "required": ["layout"] } } }, @@ -829,6 +992,7 @@ "properties": { "ssh": { "type": "object", + "additionalProperties": false, "properties": { "emit_keys_to_console": { "type": "boolean", @@ -836,7 +1000,6 @@ "description": "Set false to avoid printing SSH keys to system console. Default: ``true``." } }, - "additionalProperties": false, "required": ["emit_keys_to_console"] }, "ssh_key_console_blacklist": { @@ -860,9 +1023,11 @@ "landscape": { "type": "object", "required": ["client"], + "additionalProperties": false, "properties": { "client": { "type": "object", + "additionalProperties": true, "properties": { "url": { "type": "string", @@ -885,7 +1050,7 @@ "enum": ["debug", "info", "warning", "error", "critical"], "description": "The log level for the client. Default: ``info``." }, - "computer_tite": { + "computer_title": { "type": "string", "description": "The title of this computer." }, @@ -934,9 +1099,11 @@ "lxd": { "type": "object", "minProperties": 1, + "additionalProperties": false, "properties": { "init": { "type": "object", + "additionalProperties": false, "properties": { "network_address": { "type": "string", @@ -948,7 +1115,7 @@ }, "storage_backend": { "type": "string", - "enum": ["zfs", "dir"], + "enum": ["zfs", "dir", "lvm", "btrfs"], "default": "dir", "description": "Storage backend to use. Default: ``dir``." }, @@ -973,6 +1140,7 @@ "bridge": { "type": "object", "required": ["mode"], + "additionalProperties": false, "properties": { "mode": { "type": "string", @@ -984,6 +1152,12 @@ "description": "Name of the LXD network bridge to attach or create. Default: ``lxdbr0``.", "default": "lxdbr0" }, + "mtu": { + "type": "integer", + "description": "Bridge MTU, defaults to LXD's default value", + "default": -1, + "minimum": -1 + }, "ipv4_address": { "type": "string", "description": "IPv4 address for the bridge. If set, ``ipv4_netmask`` key required." @@ -1028,8 +1202,7 @@ } } } - }, - "additionalProperties": false + } } } }, @@ -1038,9 +1211,11 @@ "properties": { "mcollective": { "type": "object", + "additionalProperties": false, "properties": { "conf": { "type": "object", + "additionalProperties": false, "properties": { "public-cert": { "type": "string", @@ -1062,8 +1237,7 @@ } } } - }, - "additionalProperties": false + } } } }, @@ -1106,6 +1280,7 @@ }, "swap": { "type": "object", + "additionalProperties": false, "properties": { "filename": { "type": "string", @@ -1126,8 +1301,7 @@ ], "description": "The maxsize in bytes of the swap file" } - }, - "additionalProperties": false + } } } }, @@ -1136,6 +1310,7 @@ "properties": { "ntp": { "type": ["null", "object"], + "additionalProperties": false, "properties": { "pools": { "type": "array", @@ -1168,6 +1343,8 @@ "config": { "description": "Configuration settings or overrides for the\n``ntp_client`` specified.", "type": "object", + "minProperties": 1, + "additionalProperties": false, "properties": { "confpath": { "type": "string", @@ -1193,12 +1370,9 @@ "type": "string", "description": "Inline template allowing users to define their\nown ``ntp_client`` configuration template.\nThe value must start with '## template:jinja'\nto enable use of templating support.\n" } - }, - "minProperties": 1, - "additionalProperties": false + } } - }, - "additionalProperties": false + } } } }, @@ -1232,22 +1406,22 @@ "description": "Set ``true`` to reboot the system if required by presence of `/var/run/reboot-required`. Default: ``false``" }, "apt_update": { - "type": "boolean", - "default": false, - "description": "DEPRECATED. Use ``package_update``. Default: ``false``", - "deprecated": true + "type": "boolean", + "default": false, + "description": "Dropped after April 2027. Use ``package_update``. Default: ``false``", + "deprecated": true }, "apt_upgrade": { - "type": "boolean", - "default": false, - "description": "DEPRECATED. Use ``package_upgrade``. Default: ``false``", - "deprecated": true + "type": "boolean", + "default": false, + "description": "Dropped after April 2027. Use ``package_upgrade``. Default: ``false``", + "deprecated": true }, "apt_reboot_if_required": { - "type": "boolean", - "default": false, - "description": "DEPRECATED. Use ``package_reboot_if_required``. Default: ``false``", - "deprecated": true + "type": "boolean", + "default": false, + "description": "Dropped after April 2027. Use ``package_reboot_if_required``. Default: ``false``", + "deprecated": true } } }, @@ -1256,8 +1430,8 @@ "properties": { "phone_home": { "type": "object", - "additionalProperties": false, "required": ["url"], + "additionalProperties": false, "properties": { "url": { "type": "string", @@ -1299,15 +1473,20 @@ "properties": { "power_state": { "type": "object", - "additionalProperties": false, "required": ["mode"], + "additionalProperties": false, "properties": { "delay": { "description": "Time in minutes to delay after cloud-init has finished. Can be ``now`` or an integer specifying the number of minutes to delay. Default: ``now``", "default": "now", "oneOf": [ {"type": "integer", "minimum": 0}, - {"type": "string", "pattern": "^\\+?[0-9]+$"}, + { + "type": "string", + "pattern": "^\\+?[0-9]+$", + "deprecated": true, + "description": "Use of string for this value will be dropped after April 2027. Use ``now`` or integer type." + }, {"enum": ["now"]} ] }, @@ -1577,8 +1756,8 @@ {"type": "string"}, { "type": "object", - "additionalProperties": false, "required": ["content"], + "additionalProperties": false, "properties": { "filename": { "type": "string" @@ -1673,8 +1852,18 @@ "additionalProperties": false, "properties": { "enabled": { - "type": ["boolean", "string"], - "description": "Whether vendor data is enabled or not. Use of string for this value is DEPRECATED. Default: ``true``" + "description": "Whether vendor data is enabled or not. Default: ``true``", + "oneOf": [ + { + "type": "boolean", + "default": true + }, + { + "type": "string", + "description": "Use of string for this value is DEPRECATED. Use a boolean value instead.", + "deprecated": true + } + ] }, "prefix": { "type": ["array", "string"], @@ -1749,9 +1938,13 @@ "ssh_pwauth": { "oneOf": [ {"type": "boolean"}, - {"type": "string"} + { + "type": "string", + "description": "Use of non-boolean values for this field is DEPRECATED and will result in an error in a future version of cloud-init.", + "deprecated": true + } ], - "description": "Sets whether or not to accept password authentication. ``true`` will enable password auth. ``false`` will disable. Default is to leave the value unchanged. Use of non-boolean values for this field is DEPRECATED and will result in an error in a future version of cloud-init." + "description": "Sets whether or not to accept password authentication. ``true`` will enable password auth. ``false`` will disable. Default is to leave the value unchanged." }, "chpasswd": { "type": "object", @@ -1762,6 +1955,46 @@ "default": true, "description": "Whether to expire all user passwords such that a password will need to be reset on the user's next login. Default: ``true``" }, + "users": { + "description": "Replaces the deprecated ``list`` key. This key represents a list of existing users to set passwords for. Each item under users contains the following required keys: ``name`` and ``password`` or in the case of a randomly generated password, ``name`` and ``type``. The ``type`` key has a default value of ``hash``, and may alternatively be set to ``text`` or ``RANDOM``.", + "type": "array", + "items": { + "minItems": 1, + "type": "object", + "anyOf": [ + { + "required": ["name", "type"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "type": { + "enum": ["RANDOM"], + "type": "string" + } + } + }, + { + "required": ["name", "password"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "type": { + "enum": ["hash", "text"], + "default": "hash", + "type": "string" + }, + "password": { + "type": "string" + } + } + } + ] + } + }, "list": { "oneOf": [ {"type": "string"}, @@ -1773,7 +2006,8 @@ }} ], "minItems": 1, - "description": "List of ``username:password`` pairs. Each user will have the corresponding password set. A password can be randomly generated by specifying ``RANDOM`` or ``R`` as a user's password. A hashed password, created by a tool like ``mkpasswd``, can be specified. A regex (``r'\\$(1|2a|2y|5|6)(\\$.+){2}'``) is used to determine if a password value should be treated as a hash.\n\nUse of a multiline string for this field is DEPRECATED and will result in an error in a future version of cloud-init." + "description": "List of ``username:password`` pairs. Each user will have the corresponding password set. A password can be randomly generated by specifying ``RANDOM`` or ``R`` as a user's password. A hashed password, created by a tool like ``mkpasswd``, can be specified. A regex (``r'\\$(1|2a|2y|5|6)(\\$.+){2}'``) is used to determine if a password value should be treated as a hash.\n\nUse of a multiline string for this field is DEPRECATED and will result in an error in a future version of cloud-init.", + "deprecated": true } } }, @@ -1788,12 +2022,12 @@ "properties": { "snap": { "type": "object", - "additionalProperties": false, "minProperties": 1, + "additionalProperties": false, "properties": { "assertions": { - "type": ["object", "array"], "description": "Properly-signed snap assertions which will run before and snap ``commands``.", + "type": ["object", "array"], "items": {"type": "string"}, "additionalItems": false, "minItems": 1, @@ -1880,13 +2114,13 @@ "ssh_keys": { "type": "object", "description": "A dictionary entries for the public and private host keys of each desired key type. Entries in the ``ssh_keys`` config dict should have keys in the format ``_private``, ``_public``, and, optionally, ``_certificate``, e.g. ``rsa_private: ``, ``rsa_public: ``, and ``rsa_certificate: ``. Not all key types have to be specified, ones left unspecified will not be used. If this config option is used, then separate keys will not be automatically generated. In order to specify multiline private host keys and certificates, use yaml multiline syntax.", + "additionalProperties": false, "patternProperties": { "^(dsa|ecdsa|ed25519|rsa)_(public|private|certificate)$": { "label": "", "type": "string" } - }, - "additionalProperties": false + } }, "ssh_authorized_keys": { "type": "array", @@ -1965,6 +2199,8 @@ "properties": { "ubuntu_advantage": { "type": "object", + "required": ["token"], + "additionalProperties": false, "properties": { "enable": { "type": "array", @@ -1974,10 +2210,44 @@ "token": { "type": "string", "description": "Required contract token obtained from https://ubuntu.com/advantage to attach." + }, + "config": { + "type": "object", + "description": "Configuration settings or override Ubuntu Advantage config", + "properties": { + "http_proxy": { + "type": "string", + "format": "uri", + "description": "Ubuntu Advantage HTTP Proxy URL" + }, + "https_proxy": { + "type": "string", + "format": "uri", + "description": "Ubuntu Advantage HTTPS Proxy URL" + }, + "global_apt_http_proxy": { + "type": "string", + "format": "uri", + "description": "HTTP Proxy URL used for all APT repositories on a system. Stored at ``/etc/apt/apt.conf.d/90ubuntu-advantage-aptproxy``" + }, + "global_apt_https_proxy": { + "type": "string", + "format": "uri", + "description": "HTTPS Proxy URL used for all APT repositories on a system. Stored at ``/etc/apt/apt.conf.d/90ubuntu-advantage-aptproxy``" + }, + "ua_apt_http_proxy": { + "type": "string", + "format": "uri", + "description": "HTTP Proxy URL used only for Ubuntu Advantage APT repositories. Stored at ``/etc/apt/apt.conf.d/90ubuntu-advantage-aptproxy``" + }, + "ua_apt_https_proxy": { + "type": "string", + "format": "uri", + "description": "HTTPS Proxy URL used only for Ubuntu Advantage APT repositories. Stored at ``/etc/apt/apt.conf.d/90ubuntu-advantage-aptproxy``" + } + } } - }, - "required": ["token"], - "additionalProperties": false + } } } }, @@ -1990,10 +2260,10 @@ "properties": { "nvidia": { "type": "object", - "additionalProperties": false, "required": [ "license-accepted" ], + "additionalProperties": false, "properties": { "license-accepted": { "type": "boolean", @@ -2014,8 +2284,15 @@ "properties": { "manage_etc_hosts": { "default": false, - "description": "Whether to manage ``/etc/hosts`` on the system. If ``true``, render the hosts file using ``/etc/cloud/templates/hosts.tmpl`` replacing ``$hostname`` and ``$fdqn``. If ``localhost``, append a ``127.0.1.1`` entry that resolves from FQDN and hostname every boot. Default: ``false``. DEPRECATED value ``template`` will be dropped, use ``true`` instead.", - "enum": [true, false, "template", "localhost"] + "description": "Whether to manage ``/etc/hosts`` on the system. If ``true``, render the hosts file using ``/etc/cloud/templates/hosts.tmpl`` replacing ``$hostname`` and ``$fdqn``. If ``localhost``, append a ``127.0.1.1`` entry that resolves from FQDN and hostname every boot. Default: ``false``.", + "oneOf": [ + {"enum": [true, false, "localhost"]}, + { + "enum": ["template"], + "description": "Value ``template`` will be dropped after April 2027. Use ``true`` instead.", + "deprecated": true + } + ] }, "fqdn": { "type": "string", @@ -2060,7 +2337,7 @@ {"type": "string"}, {"type": "object", "$ref": "#/$defs/users_groups.user"} ], - "description": "The ``user`` dictionary values override the ``default_user`` configuration from ``/etc/cloud/cloud.cfg``. The `user` dictionary keys supported for the default_user are the same as the ``users`` schema. DEPRECATED: string and types will be removed in a future release. Use ``users`` instead." + "description": "The ``user`` dictionary values override the ``default_user`` configuration from ``/etc/cloud/cloud.cfg``. The `user` dictionary keys supported for the default_user are the same as the ``users`` schema." }, "users": { "type": ["string", "array", "object"], @@ -2075,6 +2352,49 @@ } } }, + "cc_wireguard": { + "type": "object", + "properties": { + "wireguard": { + "type": ["null", "object"], + "properties": { + "interfaces": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the interface. Typically wgx (example: wg0)" + }, + "config_path": { + "type": "string", + "description": "Path to configuration file of Wireguard interface" + }, + "content": { + "type": "string", + "description": "Wireguard interface configuration. Contains key, peer, ..." + } + }, + "additionalProperties": false + }, + "minItems": 1 + }, + "readinessprobe": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "description": "List of shell commands to be executed as probes." + } + }, + "required": ["interfaces"], + "minProperties": 1, + "additionalProperties": false + } + } + }, "cc_write_files": { "type": "object", "properties": { @@ -2082,6 +2402,8 @@ "type": "array", "items": { "type": "object", + "required": ["path"], + "additionalProperties": false, "properties": { "path": { "type": "string", @@ -2089,7 +2411,7 @@ }, "content": { "type": "string", - "default": "", + "default": "''", "description": "Optional content to write to the provided ``path``. When content is present and encoding is not 'text/plain', decode the content prior to writing. Default: ``''``" }, "owner": { @@ -2099,7 +2421,7 @@ }, "permissions": { "type": "string", - "default": "0o644", + "default": "'0o644'", "description": "Optional file permissions to set on ``path`` represented as an octal string '0###'. Default: ``0o644``" }, "encoding": { @@ -2118,9 +2440,7 @@ "default": false, "description": "Defer writing the file until 'final' stage, after users were created, and packages were installed. Default: ``false``." } - }, - "required": ["path"], - "additionalProperties": false + } }, "minItems": 1 } @@ -2137,11 +2457,13 @@ "yum_repos": { "type": "object", "minProperties": 1, + "additionalProperties": false, "patternProperties": { "^[0-9a-zA-Z -_]+$": { "label": "", "type": "object", "description": "Object keyed on unique yum repo IDs. The key used will be used to write yum repo config files in ``yum_repo_dir``/.repo.", + "additionalProperties": false, "properties": { "baseurl": { "type": "string", @@ -2171,8 +2493,7 @@ }, "required": ["baseurl"] } - }, - "additionalProperties": false + } } } }, @@ -2181,11 +2502,14 @@ "properties": { "zypper": { "type": "object", + "minProperties": 1, + "additionalProperties": true, "properties": { "repos": { "type": "array", "items": { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string", @@ -2200,8 +2524,7 @@ "required": [ "id", "baseurl" - ], - "additionalProperties": true + ] }, "minItems": 1 }, @@ -2209,22 +2532,125 @@ "type": "object", "description": "Any supported zypo.conf key is written to ``/etc/zypp/zypp.conf``" } - }, - "minProperties": 1, - "additionalProperties": false + } + } + } + }, + "reporting_config": { + "type": "object", + "properties": { + "reporting": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^.+$": { + "label": "", + "type": "object", + "oneOf": [ + { + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["log"] + }, + "level": { + "type": "string", + "enum": ["DEBUG", "INFO", "WARN", "ERROR", "FATAL"], + "default": "DEBUG" + } + } + }, + { + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["print"] + } + } + }, + { + "additionalProperties": false, + "required": ["type", "endpoint"], + "properties": { + "type": { + "type": "string", + "enum": ["webhook"] + }, + "endpoint": { + "type": "string", + "format": "uri", + "description": "The URL to send the event to." + }, + "consumer_key": { + "type": "string", + "description": "The consumer key to use for the webhook." + }, + "token_key": { + "type": "string", + "description": "The token key to use for the webhook." + }, + "token_secret": { + "type": "string", + "description": "The token secret to use for the webhook." + }, + "consumer_secret": { + "type": "string", + "description": "The consumer secret to use for the webhook." + }, + "timeout": { + "type": "number", + "minimum": 0, + "description": "The timeout in seconds to wait for a response from the webhook." + }, + "retries": { + "type": "integer", + "minimum": 0, + "description": "The number of times to retry sending the webhook." + } + } + }, + { + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["hyperv"] + }, + "kvp_file_path": { + "type": "string", + "description": "The path to the KVP file to use for the hyperv reporter.", + "default": "/var/lib/hyperv/.kvp_pool_1" + }, + "event_types": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ] + } + } } } } }, "allOf": [ + { "$ref": "#/$defs/cc_ansible" }, { "$ref": "#/$defs/cc_apk_configure" }, { "$ref": "#/$defs/cc_apt_configure" }, { "$ref": "#/$defs/cc_apt_pipelining" }, + { "$ref": "#/$defs/cc_ubuntu_autoinstall"}, { "$ref": "#/$defs/cc_bootcmd" }, { "$ref": "#/$defs/cc_byobu" }, { "$ref": "#/$defs/cc_ca_certs" }, { "$ref": "#/$defs/cc_chef" }, - { "$ref": "#/$defs/cc_debug" }, { "$ref": "#/$defs/cc_disable_ec2_metadata" }, { "$ref": "#/$defs/cc_disk_setup" }, { "$ref": "#/$defs/cc_fan" }, @@ -2266,8 +2692,10 @@ { "$ref": "#/$defs/cc_update_etc_hosts"}, { "$ref": "#/$defs/cc_update_hostname"}, { "$ref": "#/$defs/cc_users_groups"}, + { "$ref": "#/$defs/cc_wireguard"}, { "$ref": "#/$defs/cc_write_files"}, { "$ref": "#/$defs/cc_yum_add_repo"}, - { "$ref": "#/$defs/cc_zypper_add_repo"} + { "$ref": "#/$defs/cc_zypper_add_repo"}, + { "$ref": "#/$defs/reporting_config"} ] } diff --git a/cloudinit/config/schemas/versions.schema.cloud-config.json b/cloudinit/config/schemas/versions.schema.cloud-config.json index 4ff3b4d15..bca0a11ed 100644 --- a/cloudinit/config/schemas/versions.schema.cloud-config.json +++ b/cloudinit/config/schemas/versions.schema.cloud-config.json @@ -7,11 +7,11 @@ { "properties": { "version": { - "enum": ["22.2", "v1"] + "enum": [ "v1" ] } } }, - {"$ref": "./schema-cloud-config-v1.json"} + {"$ref": "https://raw.githubusercontent.com/canonical/cloud-init/main/cloudinit/config/schemas/schema-cloud-config-v1.json"} ] } ] diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index b034e2c84..4a468cf8c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -16,7 +16,7 @@ import string import urllib.parse from io import StringIO -from typing import Any, Mapping, Type +from typing import Any, Mapping, MutableMapping, Optional, Type from cloudinit import importer from cloudinit import log as logging @@ -25,6 +25,7 @@ from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES from cloudinit.net import activators, eni, network_state, renderers from cloudinit.net.network_state import parse_net_config_data +from cloudinit.net.renderer import Renderer from .networking import LinuxNetworking, Networking @@ -47,6 +48,7 @@ "fedora", "miraclelinux", "openEuler", + "openmandriva", "photon", "rhel", "rocky", @@ -75,8 +77,9 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" + default_owner = "root:root" init_cmd = ["service"] # systemctl, service etc - renderer_configs: Mapping[str, Mapping[str, Any]] = {} + renderer_configs: Mapping[str, MutableMapping[str, Any]] = {} _preferred_ntp_clients = None networking_cls: Type[Networking] = LinuxNetworking # This is used by self.shutdown_command(), and can be overridden in @@ -116,7 +119,18 @@ def _write_network(self, settings): "_write_network_config needs implementation.\n" % self.name ) - def _write_network_state(self, network_state): + @property + def network_activator(self) -> Optional[Type[activators.NetworkActivator]]: + """Return the configured network activator for this environment.""" + priority = util.get_cfg_by_path( + self._cfg, ("network", "activators"), None + ) + try: + return activators.select_activator(priority=priority) + except activators.NoActivatorException: + return None + + def _get_renderer(self) -> Renderer: priority = util.get_cfg_by_path( self._cfg, ("network", "renderers"), None ) @@ -126,6 +140,9 @@ def _write_network_state(self, network_state): "Selected renderer '%s' from priority list: %s", name, priority ) renderer = render_cls(config=self.renderer_configs.get(name)) + return renderer + + def _write_network_state(self, network_state, renderer: Renderer): renderer.render_network_state(network_state) def _find_tz_file(self, tz): @@ -228,21 +245,22 @@ def apply_network_config(self, netconfig, bring_up=False) -> bool: """ # This method is preferred to apply_network which only takes # a much less complete network config format (interfaces(5)). - network_state = parse_net_config_data(netconfig) try: - self._write_network_state(network_state) + renderer = self._get_renderer() except NotImplementedError: # backwards compat until all distros have apply_network_config return self._apply_network_from_network_config( netconfig, bring_up=bring_up ) + network_state = parse_net_config_data(netconfig, renderer=renderer) + self._write_network_state(network_state, renderer) + # Now try to bring them up if bring_up: LOG.debug("Bringing up newly configured network interfaces") - try: - network_activator = activators.select_activator() - except activators.NoActivatorException: + network_activator = self.network_activator + if not network_activator: LOG.warning( "No network activator found, not bringing up " "network interfaces" @@ -509,6 +527,15 @@ def add_user(self, name, **kwargs): if isinstance(groups, str): groups = groups.split(",") + if isinstance(groups, dict): + LOG.warning( + "DEPRECATED: The user %s has a 'groups' config value of" + " type dict which is deprecated and will be removed in a" + " future version of cloud-init. Use a comma-delimited" + " string or array instead: group1,group2.", + name, + ) + # remove any white spaces in group names, most likely # that came in as a string like: groups: group1, group2 groups = [g.strip() for g in groups] @@ -630,8 +657,16 @@ def create_user(self, name, **kwargs): self.lock_passwd(name) # Configure sudo access - if "sudo" in kwargs and kwargs["sudo"] is not False: - self.write_sudo_rules(name, kwargs["sudo"]) + if "sudo" in kwargs: + if kwargs["sudo"]: + self.write_sudo_rules(name, kwargs["sudo"]) + elif kwargs["sudo"] is False: + LOG.warning( + "DEPRECATED: The user %s has a 'sudo' config value of" + " 'false' which will be dropped after April 2027." + " Use 'null' instead.", + name, + ) # Import SSH keys if "ssh_authorized_keys" in kwargs: @@ -716,6 +751,16 @@ def set_passwd(self, user, passwd, hashed=False): return True + def chpasswd(self, plist_in: list, hashed: bool): + payload = ( + "\n".join( + (":".join([name, password]) for name, password in plist_in) + ) + + "\n" + ) + cmd = ["chpasswd"] + (["-e"] if hashed else []) + subp.subp(cmd, payload) + def ensure_sudo_dir(self, path, sudo_base="/etc/sudoers"): # Ensure the dir is included and that # it actually exists as a directory @@ -1063,7 +1108,7 @@ def _get_arch_package_mirror_info(package_mirrors, arch): return default -def fetch(name) -> Type[Distro]: +def fetch(name: str) -> Type[Distro]: locs, looked_locs = importer.find_module(name, ["", __name__], ["Distro"]) if not locs: raise ImportError( diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 0bdfef83f..2d5cfbf65 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -11,6 +11,7 @@ from cloudinit import subp, util from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf +from cloudinit.net.renderer import Renderer from cloudinit.net.renderers import RendererNotFoundError from cloudinit.settings import PER_INSTANCE @@ -61,9 +62,9 @@ def install_packages(self, pkglist): self.update_package_sources() self.package_command("", pkgs=pkglist) - def _write_network_state(self, network_state): + def _get_renderer(self) -> Renderer: try: - super()._write_network_state(network_state) + return super()._get_renderer() except RendererNotFoundError as e: # Fall back to old _write_network raise NotImplementedError from e diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index bab222b59..77e9bf112 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -1,4 +1,5 @@ import platform +from typing import List, Optional from cloudinit import distros, helpers from cloudinit import log as logging @@ -14,18 +15,19 @@ class BSD(distros.Distro): networking_cls = BSDNetworking hostname_conf_fn = "/etc/rc.conf" rc_conf_fn = "/etc/rc.conf" + default_owner = "root:wheel" # This differs from the parent Distro class, which has -P for # poweroff. shutdown_options_map = {"halt": "-H", "poweroff": "-p", "reboot": "-r"} # Set in BSD distro subclasses - group_add_cmd_prefix = [] - pkg_cmd_install_prefix = [] - pkg_cmd_remove_prefix = [] + group_add_cmd_prefix: List[str] = [] + pkg_cmd_install_prefix: List[str] = [] + pkg_cmd_remove_prefix: List[str] = [] # There is no update/upgrade on OpenBSD - pkg_cmd_update_prefix = None - pkg_cmd_upgrade_prefix = None + pkg_cmd_update_prefix: Optional[List[str]] = None + pkg_cmd_upgrade_prefix: Optional[List[str]] = None def __init__(self, name, cfg, paths): super().__init__(name, cfg, paths) @@ -133,3 +135,7 @@ def set_timezone(self, tz): def apply_locale(self, locale, out_fn=None): LOG.debug("Cannot set the locale.") + + def chpasswd(self, plist_in: list, hashed: bool): + for name, password in plist_in: + self.set_passwd(name, password, hashed=hashed) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 6dc1ad404..87f4cc9f1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -137,9 +137,9 @@ def install_packages(self, pkglist): self.update_package_sources() self.package_command("install", pkgs=pkglist) - def _write_network_state(self, network_state): + def _write_network_state(self, *args, **kwargs): _maybe_remove_legacy_eth0() - return super()._write_network_state(network_state) + return super()._write_network_state(*args, **kwargs) def _write_hostname(self, hostname, filename): conf = None diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index c0d6390f6..b3232febf 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -89,15 +89,6 @@ def add_user(self, name, **kwargs): def set_passwd(self, user, passwd, hashed=False): if hashed: hashed_pw = passwd - elif not hasattr(crypt, "METHOD_BLOWFISH"): - # crypt.METHOD_BLOWFISH comes with Python 3.7 which is available - # on NetBSD 7 and 8. - LOG.error( - "Cannot set non-encrypted password for user %s. " - "Python >= 3.7 is required.", - user, - ) - return else: method = crypt.METHOD_BLOWFISH # pylint: disable=E1101 hashed_pw = crypt.crypt(passwd, crypt.mksalt(method)) diff --git a/cloudinit/distros/openmandriva.py b/cloudinit/distros/openmandriva.py new file mode 100644 index 000000000..b4ba8439b --- /dev/null +++ b/cloudinit/distros/openmandriva.py @@ -0,0 +1,14 @@ +# Copyright (C) 2021 LinDev +# +# Author: Bernhard Rosenkraenzer +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import fedora + + +class Distro(fedora.Distro): + pass + + +# vi: ts=4 expandtab diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 0ef4e1472..c2bed1bf1 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -36,6 +36,13 @@ def local_domain(self): return dm[0] return None + @local_domain.setter + def local_domain(self, domain): + self.parse() + self._remove_option("domain") + self._contents.append(("option", ["domain", str(domain), ""])) + return domain + @property def search_domains(self): self.parse() @@ -133,13 +140,6 @@ def add_search_domain(self, search_domain): self._contents.append(("option", ["search", s_list, ""])) return flat_sds - @local_domain.setter - def local_domain(self, domain): - self.parse() - self._remove_option("domain") - self._contents.append(("option", ["domain", str(domain), ""])) - return domain - def _parse(self, contents): entries = [] for (i, line) in enumerate(contents.splitlines()): diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index 4132734ca..cb6e583e7 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -107,7 +107,7 @@ def _write_line(self, indent_string, entry, this_entry, comment): return "%s%s%s%s%s" % ( indent_string, key, - self._a_to_u("="), + "=", val, cmnt, ) diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index ec6470a94..4e75b6ecf 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -11,7 +11,6 @@ import copy -from cloudinit import util from cloudinit.distros import PREFERRED_NTP_CLIENTS, debian @@ -39,14 +38,7 @@ def __init__(self, name, cfg, paths): def preferred_ntp_clients(self): """The preferred ntp client is dependent on the version.""" if not self._preferred_ntp_clients: - (_name, _version, codename) = util.system_info()["dist"] - # Xenial cloud-init only installed ntp, UbuntuCore has timesyncd. - if codename == "xenial" and not util.system_is_snappy(): - self._preferred_ntp_clients = ["ntp"] - else: - self._preferred_ntp_clients = copy.deepcopy( - PREFERRED_NTP_CLIENTS - ) + self._preferred_ntp_clients = copy.deepcopy(PREFERRED_NTP_CLIENTS) return self._preferred_ntp_clients diff --git a/cloudinit/dmi.py b/cloudinit/dmi.py index 3a999d41c..dff9ab0fc 100644 --- a/cloudinit/dmi.py +++ b/cloudinit/dmi.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import os from collections import namedtuple +from typing import Optional from cloudinit import log as logging from cloudinit import subp @@ -11,8 +12,8 @@ # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" -kdmi = namedtuple("KernelNames", ["linux", "freebsd"]) -kdmi.__new__.defaults__ = (None, None) +KernelNames = namedtuple("KernelNames", ["linux", "freebsd"]) +KernelNames.__new__.__defaults__ = (None, None) # FreeBSD's kenv(1) and Linux /sys/class/dmi/id/* both use different names from # dmidecode. The values are the same, and ultimately what we're interested in. @@ -20,27 +21,45 @@ # This is our canonical translation table. If we add more tools on other # platforms to find dmidecode's values, their keys need to be put in here. DMIDECODE_TO_KERNEL = { - "baseboard-asset-tag": kdmi("board_asset_tag", "smbios.planar.tag"), - "baseboard-manufacturer": kdmi("board_vendor", "smbios.planar.maker"), - "baseboard-product-name": kdmi("board_name", "smbios.planar.product"), - "baseboard-serial-number": kdmi("board_serial", "smbios.planar.serial"), - "baseboard-version": kdmi("board_version", "smbios.planar.version"), - "bios-release-date": kdmi("bios_date", "smbios.bios.reldate"), - "bios-vendor": kdmi("bios_vendor", "smbios.bios.vendor"), - "bios-version": kdmi("bios_version", "smbios.bios.version"), - "chassis-asset-tag": kdmi("chassis_asset_tag", "smbios.chassis.tag"), - "chassis-manufacturer": kdmi("chassis_vendor", "smbios.chassis.maker"), - "chassis-serial-number": kdmi("chassis_serial", "smbios.chassis.serial"), - "chassis-version": kdmi("chassis_version", "smbios.chassis.version"), - "system-manufacturer": kdmi("sys_vendor", "smbios.system.maker"), - "system-product-name": kdmi("product_name", "smbios.system.product"), - "system-serial-number": kdmi("product_serial", "smbios.system.serial"), - "system-uuid": kdmi("product_uuid", "smbios.system.uuid"), - "system-version": kdmi("product_version", "smbios.system.version"), + "baseboard-asset-tag": KernelNames("board_asset_tag", "smbios.planar.tag"), + "baseboard-manufacturer": KernelNames( + "board_vendor", "smbios.planar.maker" + ), + "baseboard-product-name": KernelNames( + "board_name", "smbios.planar.product" + ), + "baseboard-serial-number": KernelNames( + "board_serial", "smbios.planar.serial" + ), + "baseboard-version": KernelNames("board_version", "smbios.planar.version"), + "bios-release-date": KernelNames("bios_date", "smbios.bios.reldate"), + "bios-vendor": KernelNames("bios_vendor", "smbios.bios.vendor"), + "bios-version": KernelNames("bios_version", "smbios.bios.version"), + "chassis-asset-tag": KernelNames( + "chassis_asset_tag", "smbios.chassis.tag" + ), + "chassis-manufacturer": KernelNames( + "chassis_vendor", "smbios.chassis.maker" + ), + "chassis-serial-number": KernelNames( + "chassis_serial", "smbios.chassis.serial" + ), + "chassis-version": KernelNames( + "chassis_version", "smbios.chassis.version" + ), + "system-manufacturer": KernelNames("sys_vendor", "smbios.system.maker"), + "system-product-name": KernelNames( + "product_name", "smbios.system.product" + ), + "system-serial-number": KernelNames( + "product_serial", "smbios.system.serial" + ), + "system-uuid": KernelNames("product_uuid", "smbios.system.uuid"), + "system-version": KernelNames("product_version", "smbios.system.version"), } -def _read_dmi_syspath(key): +def _read_dmi_syspath(key: str) -> Optional[str]: """ Reads dmi data from /sys/class/dmi/id """ @@ -78,7 +97,7 @@ def _read_dmi_syspath(key): return None -def _read_kenv(key): +def _read_kenv(key: str) -> Optional[str]: """ Reads dmi data from FreeBSD's kenv(1) """ @@ -96,12 +115,11 @@ def _read_kenv(key): return result except subp.ProcessExecutionError as e: LOG.debug("failed kenv cmd: %s\n%s", cmd, e) - return None return None -def _call_dmidecode(key, dmidecode_path): +def _call_dmidecode(key: str, dmidecode_path: str) -> Optional[str]: """ Calls out to dmidecode to get the data out. This is mostly for supporting OS's without /sys/class/dmi/id support. @@ -119,7 +137,7 @@ def _call_dmidecode(key, dmidecode_path): return None -def read_dmi_data(key): +def read_dmi_data(key: str) -> Optional[str]: """ Wrapper for reading DMI data. diff --git a/cloudinit/features.py b/cloudinit/features.py index e1116a172..6938b0030 100644 --- a/cloudinit/features.py +++ b/cloudinit/features.py @@ -49,6 +49,16 @@ directives in cloud-config. """ + +EXPIRE_APPLIES_TO_HASHED_USERS = False +""" +If ``EXPIRE_APPLIES_TO_HASHED_USERS`` is True, then when expire is set true +in cc_set_passwords, hashed passwords will be expired. Previous to 22.3, +only non-hashed passwords were expired. + +(This flag can be removed after Jammy is no longer supported.) +""" + try: # pylint: disable=wildcard-import from cloudinit.feature_overrides import * # noqa diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index 1f9caa64f..b8196cb1f 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -4,8 +4,16 @@ import os import re from errno import EACCES -from typing import Optional +from typing import Optional, Type +from cloudinit import handlers +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE +from cloudinit.templater import MISSING_JINJA_PREFIX, render_string +from cloudinit.util import b64d, json_dumps, load_file, load_json + +JUndefinedError: Type[Exception] try: from jinja2.exceptions import UndefinedError as JUndefinedError from jinja2.lexer import operator_re @@ -14,13 +22,6 @@ JUndefinedError = Exception operator_re = re.compile(r"[-.]") -from cloudinit import handlers -from cloudinit import log as logging -from cloudinit.settings import PER_ALWAYS -from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE -from cloudinit.templater import MISSING_JINJA_PREFIX, render_string -from cloudinit.util import b64d, json_dumps, load_file, load_json - LOG = logging.getLogger(__name__) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index d0db4b5bd..406d45829 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -330,7 +330,7 @@ def items(self): class Paths(persistence.CloudInitPickleMixin): _ci_pkl_version = 1 - def __init__(self, path_cfgs, ds=None): + def __init__(self, path_cfgs: dict, ds=None): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = path_cfgs.get("cloud_dir", "/var/lib/cloud") diff --git a/cloudinit/importer.py b/cloudinit/importer.py index c9fa9dc5f..ce25fe9a0 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -8,16 +8,34 @@ # # This file is part of cloud-init. See LICENSE file for license information. -import sys +import importlib +from types import ModuleType +from typing import Optional, Sequence -def import_module(module_name): - __import__(module_name) - return sys.modules[module_name] +def import_module(module_name: str) -> ModuleType: + return importlib.import_module(module_name) -def find_module(base_name: str, search_paths, required_attrs=None) -> tuple: - """Finds and imports specified modules""" +def _count_attrs( + module_name: str, attrs: Optional[Sequence[str]] = None +) -> int: + found_attrs = 0 + if not attrs: + return found_attrs + mod = importlib.import_module(module_name) + for attr in attrs: + if hasattr(mod, attr): + found_attrs += 1 + return found_attrs + + +def find_module( + base_name: str, + search_paths: Sequence[str], + required_attrs: Optional[Sequence[str]] = None, +) -> tuple: + """Finds specified modules""" if not required_attrs: required_attrs = [] # NOTE(harlowja): translate the search paths to include the base name. @@ -31,18 +49,10 @@ def find_module(base_name: str, search_paths, required_attrs=None) -> tuple: lookup_paths.append(full_path) found_paths = [] for full_path in lookup_paths: - mod = None - try: - mod = import_module(full_path) - except ImportError: - pass - if not mod: + if not importlib.util.find_spec(full_path): continue - found_attrs = 0 - for attr in required_attrs: - if hasattr(mod, attr): - found_attrs += 1 - if found_attrs == len(required_attrs): + # Check that required_attrs are all present within the module. + if _count_attrs(full_path, required_attrs) == len(required_attrs): found_paths.append(full_path) return (found_paths, lookup_paths) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ab36406a9..c952d73ba 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -250,7 +250,7 @@ def has_netfail_standby_feature(devname): return features[62] == "1" -def is_netfail_master(devname, driver=None): +def is_netfail_master(devname, driver=None) -> bool: """A device is a "netfail master" device if: - The device does NOT have the 'master' sysfs attribute @@ -928,6 +928,13 @@ def get_interfaces_by_mac(blacklist_drivers=None) -> dict: ) +def find_interface_name_from_mac(mac: str) -> Optional[str]: + for interface_mac, interface_name in get_interfaces_by_mac().items(): + if mac.lower() == interface_mac.lower(): + return interface_name + return None + + def get_interfaces_by_mac_on_freebsd(blacklist_drivers=None) -> dict: (out, _) = subp.subp(["ifconfig", "-a", "ether"]) @@ -992,7 +999,7 @@ def get_interfaces_by_mac_on_linux(blacklist_drivers=None) -> dict: """Build a dictionary of tuples {mac: name}. Bridges and any devices that have a 'stolen' mac are excluded.""" - ret = {} + ret: dict = {} for name, mac, _driver, _devid in get_interfaces( blacklist_drivers=blacklist_drivers ): @@ -1139,8 +1146,9 @@ def has_url_connectivity(url_data: Dict[str, Any]) -> bool: return True -def network_validator(check_cb: Callable, address: str, **kwargs) -> bool: - """Use a function to determine whether address meets criteria. +def maybe_get_address(convert_to_address: Callable, address: str, **kwargs): + """Use a function to return an address. If conversion throws a ValueError + exception return False. :param check_cb: Test function, must return a truthy value @@ -1148,11 +1156,11 @@ def network_validator(check_cb: Callable, address: str, **kwargs) -> bool: The string to test. :return: - A bool indicating if the string passed the test. + Address or False """ try: - return bool(check_cb(address, **kwargs)) + return convert_to_address(address, **kwargs) except ValueError: return False @@ -1166,7 +1174,7 @@ def is_ip_address(address: str) -> bool: :return: A bool indicating if the string is an IP address or not. """ - return network_validator(ipaddress.ip_address, address) + return bool(maybe_get_address(ipaddress.ip_address, address)) def is_ipv4_address(address: str) -> bool: @@ -1178,7 +1186,7 @@ def is_ipv4_address(address: str) -> bool: :return: A bool indicating if the string is an IPv4 address or not. """ - return network_validator(ipaddress.IPv4Address, address) + return bool(maybe_get_address(ipaddress.IPv4Address, address)) def is_ipv6_address(address: str) -> bool: @@ -1190,7 +1198,7 @@ def is_ipv6_address(address: str) -> bool: :return: A bool indicating if the string is an IPv4 address or not. """ - return network_validator(ipaddress.IPv6Address, address) + return bool(maybe_get_address(ipaddress.IPv6Address, address)) def is_ip_network(address: str) -> bool: @@ -1202,7 +1210,7 @@ def is_ip_network(address: str) -> bool: :return: A bool indicating if the string is an IPv4 address or not. """ - return network_validator(ipaddress.ip_network, address, strict=False) + return bool(maybe_get_address(ipaddress.ip_network, address, strict=False)) def is_ipv4_network(address: str) -> bool: @@ -1214,7 +1222,9 @@ def is_ipv4_network(address: str) -> bool: :return: A bool indicating if the string is an IPv4 address or not. """ - return network_validator(ipaddress.IPv4Network, address, strict=False) + return bool( + maybe_get_address(ipaddress.IPv4Network, address, strict=False) + ) def is_ipv6_network(address: str) -> bool: @@ -1226,7 +1236,9 @@ def is_ipv6_network(address: str) -> bool: :return: A bool indicating if the string is an IPv4 address or not. """ - return network_validator(ipaddress.IPv6Network, address, strict=False) + return bool( + maybe_get_address(ipaddress.IPv6Network, address, strict=False) + ) def subnet_is_ipv6(subnet) -> bool: @@ -1304,265 +1316,5 @@ def mask_and_ipv4_to_bcast_addr(mask: str, ip: str) -> str: ) -class EphemeralIPv4Network(object): - """Context manager which sets up temporary static network configuration. - - No operations are performed if the provided interface already has the - specified configuration. - This can be verified with the connectivity_url_data. - If unconnected, bring up the interface with valid ip, prefix and broadcast. - If router is provided setup a default route for that interface. Upon - context exit, clean up the interface leaving no configuration behind. - """ - - def __init__( - self, - interface, - ip, - prefix_or_mask, - broadcast, - router=None, - connectivity_url_data: Dict[str, Any] = None, - static_routes=None, - ): - """Setup context manager and validate call signature. - - @param interface: Name of the network interface to bring up. - @param ip: IP address to assign to the interface. - @param prefix_or_mask: Either netmask of the format X.X.X.X or an int - prefix. - @param broadcast: Broadcast address for the IPv4 network. - @param router: Optionally the default gateway IP. - @param connectivity_url_data: Optionally, a URL to verify if a usable - connection already exists. - @param static_routes: Optionally a list of static routes from DHCP - """ - if not all([interface, ip, prefix_or_mask, broadcast]): - raise ValueError( - "Cannot init network on {0} with {1}/{2} and bcast {3}".format( - interface, ip, prefix_or_mask, broadcast - ) - ) - try: - self.prefix = ipv4_mask_to_net_prefix(prefix_or_mask) - except ValueError as e: - raise ValueError( - "Cannot setup network, invalid prefix or " - "netmask: {0}".format(e) - ) from e - - self.connectivity_url_data = connectivity_url_data - self.interface = interface - self.ip = ip - self.broadcast = broadcast - self.router = router - self.static_routes = static_routes - self.cleanup_cmds = [] # List of commands to run to cleanup state. - - def __enter__(self): - """Perform ephemeral network setup if interface is not connected.""" - if self.connectivity_url_data: - if has_url_connectivity(self.connectivity_url_data): - LOG.debug( - "Skip ephemeral network setup, instance has connectivity" - " to %s", - self.connectivity_url_data["url"], - ) - return - - self._bringup_device() - - # rfc3442 requires us to ignore the router config *if* classless static - # routes are provided. - # - # https://tools.ietf.org/html/rfc3442 - # - # If the DHCP server returns both a Classless Static Routes option and - # a Router option, the DHCP client MUST ignore the Router option. - # - # Similarly, if the DHCP server returns both a Classless Static Routes - # option and a Static Routes option, the DHCP client MUST ignore the - # Static Routes option. - if self.static_routes: - self._bringup_static_routes() - elif self.router: - self._bringup_router() - - def __exit__(self, excp_type, excp_value, excp_traceback): - """Teardown anything we set up.""" - for cmd in self.cleanup_cmds: - subp.subp(cmd, capture=True) - - def _delete_address(self, address, prefix): - """Perform the ip command to remove the specified address.""" - subp.subp( - [ - "ip", - "-family", - "inet", - "addr", - "del", - "%s/%s" % (address, prefix), - "dev", - self.interface, - ], - capture=True, - ) - - def _bringup_device(self): - """Perform the ip comands to fully setup the device.""" - cidr = "{0}/{1}".format(self.ip, self.prefix) - LOG.debug( - "Attempting setup of ephemeral network on %s with %s brd %s", - self.interface, - cidr, - self.broadcast, - ) - try: - subp.subp( - [ - "ip", - "-family", - "inet", - "addr", - "add", - cidr, - "broadcast", - self.broadcast, - "dev", - self.interface, - ], - capture=True, - update_env={"LANG": "C"}, - ) - except subp.ProcessExecutionError as e: - if "File exists" not in e.stderr: - raise - LOG.debug( - "Skip ephemeral network setup, %s already has address %s", - self.interface, - self.ip, - ) - else: - # Address creation success, bring up device and queue cleanup - subp.subp( - [ - "ip", - "-family", - "inet", - "link", - "set", - "dev", - self.interface, - "up", - ], - capture=True, - ) - self.cleanup_cmds.append( - [ - "ip", - "-family", - "inet", - "link", - "set", - "dev", - self.interface, - "down", - ] - ) - self.cleanup_cmds.append( - [ - "ip", - "-family", - "inet", - "addr", - "del", - cidr, - "dev", - self.interface, - ] - ) - - def _bringup_static_routes(self): - # static_routes = [("169.254.169.254/32", "130.56.248.255"), - # ("0.0.0.0/0", "130.56.240.1")] - for net_address, gateway in self.static_routes: - via_arg = [] - if gateway != "0.0.0.0": - via_arg = ["via", gateway] - subp.subp( - ["ip", "-4", "route", "append", net_address] - + via_arg - + ["dev", self.interface], - capture=True, - ) - self.cleanup_cmds.insert( - 0, - ["ip", "-4", "route", "del", net_address] - + via_arg - + ["dev", self.interface], - ) - - def _bringup_router(self): - """Perform the ip commands to fully setup the router if needed.""" - # Check if a default route exists and exit if it does - out, _ = subp.subp(["ip", "route", "show", "0.0.0.0/0"], capture=True) - if "default" in out: - LOG.debug( - "Skip ephemeral route setup. %s already has default route: %s", - self.interface, - out.strip(), - ) - return - subp.subp( - [ - "ip", - "-4", - "route", - "add", - self.router, - "dev", - self.interface, - "src", - self.ip, - ], - capture=True, - ) - self.cleanup_cmds.insert( - 0, - [ - "ip", - "-4", - "route", - "del", - self.router, - "dev", - self.interface, - "src", - self.ip, - ], - ) - subp.subp( - [ - "ip", - "-4", - "route", - "add", - "default", - "via", - self.router, - "dev", - self.interface, - ], - capture=True, - ) - self.cleanup_cmds.insert( - 0, ["ip", "-4", "route", "del", "default", "dev", self.interface] - ) - - class RendererNotFoundError(RuntimeError): pass - - -# vi: ts=4 expandtab diff --git a/cloudinit/net/activators.py b/cloudinit/net/activators.py index f2cc078f2..b6af37704 100644 --- a/cloudinit/net/activators.py +++ b/cloudinit/net/activators.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging from abc import ABC, abstractmethod -from typing import Iterable, List, Type +from typing import Dict, Iterable, List, Optional, Type, Union from cloudinit import subp, util from cloudinit.net.eni import available as eni_available @@ -32,7 +32,7 @@ def _alter_interface(cmd, device_name) -> bool: class NetworkActivator(ABC): @staticmethod @abstractmethod - def available() -> bool: + def available(target: Optional[str] = None) -> bool: """Return True if activator is available, otherwise return False.""" raise NotImplementedError() @@ -97,7 +97,7 @@ class IfUpDownActivator(NetworkActivator): # E.g., NetworkManager has a ifupdown plugin that requires the name # of a specific connection. @staticmethod - def available(target=None) -> bool: + def available(target: str = None) -> bool: """Return true if ifupdown can be used on this system.""" return eni_available(target=target) @@ -254,33 +254,43 @@ def bring_down_interface(device_name: str) -> bool: # This section is mostly copied and pasted from renderers.py. An abstract # version to encompass both seems overkill at this point DEFAULT_PRIORITY = [ - IfUpDownActivator, - NetplanActivator, - NetworkManagerActivator, - NetworkdActivator, + "eni", + "netplan", + "network-manager", + "networkd", ] +NAME_TO_ACTIVATOR: Dict[str, Type[NetworkActivator]] = { + "eni": IfUpDownActivator, + "netplan": NetplanActivator, + "network-manager": NetworkManagerActivator, + "networkd": NetworkdActivator, +} + def search_activator( - priority=None, target=None + priority: List[str], target: Union[str, None] ) -> List[Type[NetworkActivator]]: - if priority is None: - priority = DEFAULT_PRIORITY - unknown = [i for i in priority if i not in DEFAULT_PRIORITY] if unknown: raise ValueError( "Unknown activators provided in priority list: %s" % unknown ) - - return [activator for activator in priority if activator.available(target)] + activator_classes = [NAME_TO_ACTIVATOR[name] for name in priority] + return [ + activator_cls + for activator_cls in activator_classes + if activator_cls.available(target) + ] -def select_activator(priority=None, target=None) -> Type[NetworkActivator]: +def select_activator( + priority: Optional[List[str]] = None, target: Optional[str] = None +) -> Type[NetworkActivator]: + if priority is None: + priority = DEFAULT_PRIORITY found = search_activator(priority, target) if not found: - if priority is None: - priority = DEFAULT_PRIORITY tmsg = "" if target and target != "/": tmsg = " in target=%s" % target @@ -289,5 +299,7 @@ def select_activator(priority=None, target=None) -> Type[NetworkActivator]: "through list: %s" % (tmsg, priority) ) selected = found[0] - LOG.debug("Using selected activator: %s", selected) + LOG.debug( + "Using selected activator: %s from priority: %s", selected, priority + ) return selected diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index ff5c74131..e0f183666 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -1,11 +1,13 @@ # This file is part of cloud-init. See LICENSE file for license information. import re +from typing import Optional from cloudinit import log as logging from cloudinit import net, subp, util from cloudinit.distros import bsd_utils from cloudinit.distros.parsers.resolv_conf import ResolvConf +from cloudinit.net.network_state import NetworkState from . import renderer @@ -156,7 +158,12 @@ def _resolve_conf(self, settings): 0o644, ) - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: if target: self.target = target self._ifconfig_entries(settings=network_state) diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 53f8c6864..fd1d42562 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -10,18 +10,11 @@ import signal import time from io import StringIO -from typing import Any, Dict import configobj from cloudinit import subp, temp_utils, util -from cloudinit.net import ( - EphemeralIPv4Network, - find_fallback_nic, - get_devicelist, - has_url_connectivity, - mask_and_ipv4_to_bcast_addr, -) +from cloudinit.net import find_fallback_nic, get_devicelist LOG = logging.getLogger(__name__) @@ -48,111 +41,6 @@ class NoDHCPLeaseMissingDhclientError(NoDHCPLeaseError): """Raised when unable to find dhclient.""" -class EphemeralDHCPv4(object): - def __init__( - self, - iface=None, - connectivity_url_data: Dict[str, Any] = None, - dhcp_log_func=None, - ): - self.iface = iface - self._ephipv4 = None - self.lease = None - self.dhcp_log_func = dhcp_log_func - self.connectivity_url_data = connectivity_url_data - - def __enter__(self): - """Setup sandboxed dhcp context, unless connectivity_url can already be - reached.""" - if self.connectivity_url_data: - if has_url_connectivity(self.connectivity_url_data): - LOG.debug( - "Skip ephemeral DHCP setup, instance has connectivity" - " to %s", - self.connectivity_url_data, - ) - return - return self.obtain_lease() - - def __exit__(self, excp_type, excp_value, excp_traceback): - """Teardown sandboxed dhcp context.""" - self.clean_network() - - def clean_network(self): - """Exit _ephipv4 context to teardown of ip configuration performed.""" - if self.lease: - self.lease = None - if not self._ephipv4: - return - self._ephipv4.__exit__(None, None, None) - - def obtain_lease(self): - """Perform dhcp discovery in a sandboxed environment if possible. - - @return: A dict representing dhcp options on the most recent lease - obtained from the dhclient discovery if run, otherwise an error - is raised. - - @raises: NoDHCPLeaseError if no leases could be obtained. - """ - if self.lease: - return self.lease - leases = maybe_perform_dhcp_discovery(self.iface, self.dhcp_log_func) - if not leases: - raise NoDHCPLeaseError() - self.lease = leases[-1] - LOG.debug( - "Received dhcp lease on %s for %s/%s", - self.lease["interface"], - self.lease["fixed-address"], - self.lease["subnet-mask"], - ) - nmap = { - "interface": "interface", - "ip": "fixed-address", - "prefix_or_mask": "subnet-mask", - "broadcast": "broadcast-address", - "static_routes": [ - "rfc3442-classless-static-routes", - "classless-static-routes", - ], - "router": "routers", - } - kwargs = self.extract_dhcp_options_mapping(nmap) - if not kwargs["broadcast"]: - kwargs["broadcast"] = mask_and_ipv4_to_bcast_addr( - kwargs["prefix_or_mask"], kwargs["ip"] - ) - if kwargs["static_routes"]: - kwargs["static_routes"] = parse_static_routes( - kwargs["static_routes"] - ) - if self.connectivity_url_data: - kwargs["connectivity_url_data"] = self.connectivity_url_data - ephipv4 = EphemeralIPv4Network(**kwargs) - ephipv4.__enter__() - self._ephipv4 = ephipv4 - return self.lease - - def extract_dhcp_options_mapping(self, nmap): - result = {} - for internal_reference, lease_option_names in nmap.items(): - if isinstance(lease_option_names, list): - self.get_first_option_value( - internal_reference, lease_option_names, result - ) - else: - result[internal_reference] = self.lease.get(lease_option_names) - return result - - def get_first_option_value( - self, internal_mapping, lease_option_names, result - ): - for different_names in lease_option_names: - if not result.get(internal_mapping): - result[internal_mapping] = self.lease.get(different_names) - - def maybe_perform_dhcp_discovery(nic=None, dhcp_log_func=None): """Perform dhcp discovery if nic valid and dhclient command exists. diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b0ec67bd5..ea0b8e4aa 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -4,10 +4,12 @@ import glob import os import re +from typing import Optional from cloudinit import log as logging from cloudinit import subp, util from cloudinit.net import subnet_is_ipv6 +from cloudinit.net.network_state import NetworkState from . import ParserError, renderer @@ -561,7 +563,12 @@ def _render_interfaces(self, network_state, render_hwaddress=False): return "\n\n".join(["\n".join(s) for s in sections]) + "\n" - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" diff --git a/cloudinit/net/ephemeral.py b/cloudinit/net/ephemeral.py new file mode 100644 index 000000000..81f7079f5 --- /dev/null +++ b/cloudinit/net/ephemeral.py @@ -0,0 +1,445 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Module for ephemeral network context managers +""" +import contextlib +import logging +from typing import Any, Dict, List + +import cloudinit.net as net +from cloudinit import subp +from cloudinit.net.dhcp import ( + NoDHCPLeaseError, + maybe_perform_dhcp_discovery, + parse_static_routes, +) + +LOG = logging.getLogger(__name__) + + +class EphemeralIPv4Network(object): + """Context manager which sets up temporary static network configuration. + + No operations are performed if the provided interface already has the + specified configuration. + This can be verified with the connectivity_url_data. + If unconnected, bring up the interface with valid ip, prefix and broadcast. + If router is provided setup a default route for that interface. Upon + context exit, clean up the interface leaving no configuration behind. + """ + + def __init__( + self, + interface, + ip, + prefix_or_mask, + broadcast, + router=None, + connectivity_url_data: Dict[str, Any] = None, + static_routes=None, + ): + """Setup context manager and validate call signature. + + @param interface: Name of the network interface to bring up. + @param ip: IP address to assign to the interface. + @param prefix_or_mask: Either netmask of the format X.X.X.X or an int + prefix. + @param broadcast: Broadcast address for the IPv4 network. + @param router: Optionally the default gateway IP. + @param connectivity_url_data: Optionally, a URL to verify if a usable + connection already exists. + @param static_routes: Optionally a list of static routes from DHCP + """ + if not all([interface, ip, prefix_or_mask, broadcast]): + raise ValueError( + "Cannot init network on {0} with {1}/{2} and bcast {3}".format( + interface, ip, prefix_or_mask, broadcast + ) + ) + try: + self.prefix = net.ipv4_mask_to_net_prefix(prefix_or_mask) + except ValueError as e: + raise ValueError( + "Cannot setup network, invalid prefix or " + "netmask: {0}".format(e) + ) from e + + self.connectivity_url_data = connectivity_url_data + self.interface = interface + self.ip = ip + self.broadcast = broadcast + self.router = router + self.static_routes = static_routes + # List of commands to run to cleanup state. + self.cleanup_cmds: List[str] = [] + + def __enter__(self): + """Perform ephemeral network setup if interface is not connected.""" + if self.connectivity_url_data: + if net.has_url_connectivity(self.connectivity_url_data): + LOG.debug( + "Skip ephemeral network setup, instance has connectivity" + " to %s", + self.connectivity_url_data["url"], + ) + return + + self._bringup_device() + + # rfc3442 requires us to ignore the router config *if* classless static + # routes are provided. + # + # https://tools.ietf.org/html/rfc3442 + # + # If the DHCP server returns both a Classless Static Routes option and + # a Router option, the DHCP client MUST ignore the Router option. + # + # Similarly, if the DHCP server returns both a Classless Static Routes + # option and a Static Routes option, the DHCP client MUST ignore the + # Static Routes option. + if self.static_routes: + self._bringup_static_routes() + elif self.router: + self._bringup_router() + + def __exit__(self, excp_type, excp_value, excp_traceback): + """Teardown anything we set up.""" + for cmd in self.cleanup_cmds: + subp.subp(cmd, capture=True) + + def _delete_address(self, address, prefix): + """Perform the ip command to remove the specified address.""" + subp.subp( + [ + "ip", + "-family", + "inet", + "addr", + "del", + "%s/%s" % (address, prefix), + "dev", + self.interface, + ], + capture=True, + ) + + def _bringup_device(self): + """Perform the ip comands to fully setup the device.""" + cidr = "{0}/{1}".format(self.ip, self.prefix) + LOG.debug( + "Attempting setup of ephemeral network on %s with %s brd %s", + self.interface, + cidr, + self.broadcast, + ) + try: + subp.subp( + [ + "ip", + "-family", + "inet", + "addr", + "add", + cidr, + "broadcast", + self.broadcast, + "dev", + self.interface, + ], + capture=True, + update_env={"LANG": "C"}, + ) + except subp.ProcessExecutionError as e: + if "File exists" not in str(e.stderr): + raise + LOG.debug( + "Skip ephemeral network setup, %s already has address %s", + self.interface, + self.ip, + ) + else: + # Address creation success, bring up device and queue cleanup + subp.subp( + [ + "ip", + "-family", + "inet", + "link", + "set", + "dev", + self.interface, + "up", + ], + capture=True, + ) + self.cleanup_cmds.append( + [ + "ip", + "-family", + "inet", + "link", + "set", + "dev", + self.interface, + "down", + ] + ) + self.cleanup_cmds.append( + [ + "ip", + "-family", + "inet", + "addr", + "del", + cidr, + "dev", + self.interface, + ] + ) + + def _bringup_static_routes(self): + # static_routes = [("169.254.169.254/32", "130.56.248.255"), + # ("0.0.0.0/0", "130.56.240.1")] + for net_address, gateway in self.static_routes: + via_arg = [] + if gateway != "0.0.0.0": + via_arg = ["via", gateway] + subp.subp( + ["ip", "-4", "route", "append", net_address] + + via_arg + + ["dev", self.interface], + capture=True, + ) + self.cleanup_cmds.insert( + 0, + ["ip", "-4", "route", "del", net_address] + + via_arg + + ["dev", self.interface], + ) + + def _bringup_router(self): + """Perform the ip commands to fully setup the router if needed.""" + # Check if a default route exists and exit if it does + out, _ = subp.subp(["ip", "route", "show", "0.0.0.0/0"], capture=True) + if "default" in out: + LOG.debug( + "Skip ephemeral route setup. %s already has default route: %s", + self.interface, + out.strip(), + ) + return + subp.subp( + [ + "ip", + "-4", + "route", + "add", + self.router, + "dev", + self.interface, + "src", + self.ip, + ], + capture=True, + ) + self.cleanup_cmds.insert( + 0, + [ + "ip", + "-4", + "route", + "del", + self.router, + "dev", + self.interface, + "src", + self.ip, + ], + ) + subp.subp( + [ + "ip", + "-4", + "route", + "add", + "default", + "via", + self.router, + "dev", + self.interface, + ], + capture=True, + ) + self.cleanup_cmds.insert( + 0, ["ip", "-4", "route", "del", "default", "dev", self.interface] + ) + + +class EphemeralIPv6Network: + """Context manager which sets up a ipv6 link local address + + The linux kernel assigns link local addresses on link-up, which is + sufficient for link-local communication. + """ + + def __init__(self, interface): + """Setup context manager and validate call signature. + + @param interface: Name of the network interface to bring up. + @param ip: IP address to assign to the interface. + @param prefix: IPv6 uses prefixes, not netmasks + """ + if not interface: + raise ValueError("Cannot init network on {0}".format(interface)) + + self.interface = interface + + def __enter__(self): + """linux kernel does autoconfiguration even when autoconf=0 + + https://www.kernel.org/doc/html/latest/networking/ipv6.html + """ + if net.read_sys_net(self.interface, "operstate") != "up": + subp.subp( + ["ip", "link", "set", "dev", self.interface, "up"], + capture=False, + ) + + def __exit__(self, *_args): + """No need to set the link to down state""" + + +class EphemeralDHCPv4(object): + def __init__( + self, + iface=None, + connectivity_url_data: Dict[str, Any] = None, + dhcp_log_func=None, + ): + self.iface = iface + self._ephipv4 = None + self.lease = None + self.dhcp_log_func = dhcp_log_func + self.connectivity_url_data = connectivity_url_data + + def __enter__(self): + """Setup sandboxed dhcp context, unless connectivity_url can already be + reached.""" + if self.connectivity_url_data: + if net.has_url_connectivity(self.connectivity_url_data): + LOG.debug( + "Skip ephemeral DHCP setup, instance has connectivity" + " to %s", + self.connectivity_url_data, + ) + return + return self.obtain_lease() + + def __exit__(self, excp_type, excp_value, excp_traceback): + """Teardown sandboxed dhcp context.""" + self.clean_network() + + def clean_network(self): + """Exit _ephipv4 context to teardown of ip configuration performed.""" + if self.lease: + self.lease = None + if not self._ephipv4: + return + self._ephipv4.__exit__(None, None, None) + + def obtain_lease(self): + """Perform dhcp discovery in a sandboxed environment if possible. + + @return: A dict representing dhcp options on the most recent lease + obtained from the dhclient discovery if run, otherwise an error + is raised. + + @raises: NoDHCPLeaseError if no leases could be obtained. + """ + if self.lease: + return self.lease + leases = maybe_perform_dhcp_discovery(self.iface, self.dhcp_log_func) + if not leases: + raise NoDHCPLeaseError() + self.lease = leases[-1] + LOG.debug( + "Received dhcp lease on %s for %s/%s", + self.lease["interface"], + self.lease["fixed-address"], + self.lease["subnet-mask"], + ) + nmap = { + "interface": "interface", + "ip": "fixed-address", + "prefix_or_mask": "subnet-mask", + "broadcast": "broadcast-address", + "static_routes": [ + "rfc3442-classless-static-routes", + "classless-static-routes", + ], + "router": "routers", + } + kwargs = self.extract_dhcp_options_mapping(nmap) + if not kwargs["broadcast"]: + kwargs["broadcast"] = net.mask_and_ipv4_to_bcast_addr( + kwargs["prefix_or_mask"], kwargs["ip"] + ) + if kwargs["static_routes"]: + kwargs["static_routes"] = parse_static_routes( + kwargs["static_routes"] + ) + if self.connectivity_url_data: + kwargs["connectivity_url_data"] = self.connectivity_url_data + ephipv4 = EphemeralIPv4Network(**kwargs) + ephipv4.__enter__() + self._ephipv4 = ephipv4 + return self.lease + + def extract_dhcp_options_mapping(self, nmap): + result = {} + for internal_reference, lease_option_names in nmap.items(): + if isinstance(lease_option_names, list): + self.get_first_option_value( + internal_reference, lease_option_names, result + ) + else: + result[internal_reference] = self.lease.get(lease_option_names) + return result + + def get_first_option_value( + self, internal_mapping, lease_option_names, result + ): + for different_names in lease_option_names: + if not result.get(internal_mapping): + result[internal_mapping] = self.lease.get(different_names) + + +class EphemeralIPNetwork: + """Marries together IPv4 and IPv6 ephemeral context managers""" + + def __init__(self, interface, ipv6: bool = False, ipv4: bool = True): + self.interface = interface + self.ipv4 = ipv4 + self.ipv6 = ipv6 + self.stack = contextlib.ExitStack() + self.state_msg: str = "" + + def __enter__(self): + # ipv6 dualstack might succeed when dhcp4 fails + # therefore catch exception unless only v4 is used + try: + if self.ipv4: + self.stack.enter_context(EphemeralDHCPv4(self.interface)) + if self.ipv6: + self.stack.enter_context(EphemeralIPv6Network(self.interface)) + # v6 link local might be usable + # caller may want to log network state + except NoDHCPLeaseError as e: + if self.ipv6: + self.state_msg = "using link-local ipv6" + else: + raise e + return self + + def __exit__(self, *_args): + self.stack.close() diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 2af0ee9ba..7b91077d4 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -2,6 +2,8 @@ import copy import os +import textwrap +from typing import Optional, cast from cloudinit import log as logging from cloudinit import safeyaml, subp, util @@ -238,7 +240,12 @@ def features(self): LOG.debug("Failed to list features from netplan info: %s", e) return self._features - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: # check network state for version # if v2, then extract network_state.config # else render_v2_from_state @@ -274,12 +281,27 @@ def _net_setup_link(self, run=False): LOG.debug("netplan net_setup_link postcmd disabled") return setup_lnk = ["udevadm", "test-builtin", "net_setup_link"] - for cmd in [ - setup_lnk + [SYS_CLASS_NET + iface] - for iface in get_devicelist() - if os.path.islink(SYS_CLASS_NET + iface) - ]: - subp.subp(cmd, capture=True) + + # It's possible we can race a udev rename and attempt to run + # net_setup_link on a device that no longer exists. When this happens, + # we don't know what the device was renamed to, so re-gather the + # entire list of devices and try again. + last_exception = Exception + for _ in range(5): + try: + for iface in get_devicelist(): + if os.path.islink(SYS_CLASS_NET + iface): + subp.subp( + setup_lnk + [SYS_CLASS_NET + iface], capture=True + ) + break + except subp.ProcessExecutionError as e: + last_exception = e + else: + raise RuntimeError( + "'udevadm test-builtin net_setup_link' unable to run " + "successfully for all devices." + ) from last_exception def _render_content(self, network_state: NetworkState): @@ -293,7 +315,7 @@ def _render_content(self, network_state: NetworkState): ) ethernets = {} - wifis = {} + wifis: dict = {} bridges = {} bonds = {} vlans = {} @@ -335,8 +357,11 @@ def _render_content(self, network_state: NetworkState): bond = {} bond_config = {} # extract bond params and drop the bond_ prefix as it's - # redundent in v2 yaml format - v2_bond_map = NET_CONFIG_TO_V2.get("bond") + # redundant in v2 yaml format + v2_bond_map = cast(dict, NET_CONFIG_TO_V2.get("bond")) + # Previous cast is needed to help mypy to know that the key is + # present in `NET_CONFIG_TO_V2`. This could probably be removed + # by using `Literal` when supported. for match in ["bond_", "bond-"]: bond_params = _get_params_dict_by_match(ifcfg, match) for (param, value) in bond_params.items(): @@ -348,7 +373,7 @@ def _render_content(self, network_state: NetworkState): if len(bond_config) > 0: bond.update({"parameters": bond_config}) if ifcfg.get("mac_address"): - bond["macaddress"] = ifcfg.get("mac_address").lower() + bond["macaddress"] = ifcfg["mac_address"].lower() slave_interfaces = ifcfg.get("bond-slaves") if slave_interfaces == "none": _extract_bond_slaves_by_name(interfaces, bond, ifname) @@ -357,19 +382,24 @@ def _render_content(self, network_state: NetworkState): elif if_type == "bridge": # required_keys = ['name', 'bridge_ports'] - ports = sorted(copy.copy(ifcfg.get("bridge_ports"))) - bridge = { + bridge_ports = ifcfg.get("bridge_ports") + # mypy wrong error. `copy(None)` is supported: + ports = sorted(copy.copy(bridge_ports)) # type: ignore + bridge: dict = { "interfaces": ports, } # extract bridge params and drop the bridge prefix as it's - # redundent in v2 yaml format + # redundant in v2 yaml format match_prefix = "bridge_" params = _get_params_dict_by_match(ifcfg, match_prefix) br_config = {} # v2 yaml uses different names for the keys # and at least one value format change - v2_bridge_map = NET_CONFIG_TO_V2.get("bridge") + v2_bridge_map = cast(dict, NET_CONFIG_TO_V2.get("bridge")) + # Previous cast is needed to help mypy to know that the key is + # present in `NET_CONFIG_TO_V2`. This could probably be removed + # by using `Literal` when supported. for (param, value) in params.items(): newname = v2_bridge_map.get(param) if newname is None: @@ -386,7 +416,7 @@ def _render_content(self, network_state: NetworkState): if len(br_config) > 0: bridge.update({"parameters": br_config}) if ifcfg.get("mac_address"): - bridge["macaddress"] = ifcfg.get("mac_address").lower() + bridge["macaddress"] = ifcfg["mac_address"].lower() _extract_addresses(ifcfg, bridge, ifname, self.features) bridges.update({ifname: bridge}) @@ -421,7 +451,7 @@ def _render_section(name, section): explicit_end=False, noalias=True, ) - txt = util.indent(dump, " " * 4) + txt = textwrap.indent(dump, " " * 4) return [txt] return [] diff --git a/cloudinit/net/network_manager.py b/cloudinit/net/network_manager.py index 8fd155754..8053511cb 100644 --- a/cloudinit/net/network_manager.py +++ b/cloudinit/net/network_manager.py @@ -11,10 +11,12 @@ import itertools import os import uuid +from typing import Optional from cloudinit import log as logging from cloudinit import subp, util from cloudinit.net import is_ipv6_address, subnet_is_ipv6 +from cloudinit.net.network_state import NetworkState from . import renderer @@ -69,7 +71,7 @@ def _set_ip_method(self, family, subnet_type): method_map = { "static": "manual", - "dhcp6": "dhcp", + "dhcp6": "auto", "ipv6_slaac": "auto", "ipv6_dhcpv6-stateless": "auto", "ipv6_dhcpv6-stateful": "auto", @@ -96,8 +98,6 @@ def _set_ip_method(self, family, subnet_type): self.config[family]["method"] = method self._set_default(family, "may-fail", "false") - if family == "ipv6": - self._set_default(family, "addr-gen-mode", "stable-privacy") def _add_numbered(self, section, key_prefix, value): """ @@ -344,7 +344,12 @@ def con_ref(self, con_id): # Well, what can we do... return con_id - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: # First pass makes sure there's NMConnections for all known # interfaces that have UUIDs that can be linked to from related # interfaces diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 3c7ee5a3f..e4f7a7fdc 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -7,9 +7,11 @@ import copy import functools import logging +from typing import TYPE_CHECKING, Any, Dict, Optional from cloudinit import safeyaml, util from cloudinit.net import ( + find_interface_name_from_mac, get_interfaces_by_mac, ipv4_mask_to_net_prefix, ipv6_mask_to_net_prefix, @@ -20,6 +22,9 @@ net_prefix_to_ipv4_mask, ) +if TYPE_CHECKING: + from cloudinit.net.renderer import Renderer + LOG = logging.getLogger(__name__) NETWORK_STATE_VERSION = 1 @@ -44,7 +49,7 @@ "accept-ra", ] -NET_CONFIG_TO_V2 = { +NET_CONFIG_TO_V2: Dict[str, Dict[str, Any]] = { "bond": { "bond-ad-select": "ad-select", "bond-arp-interval": "arp-interval", @@ -56,7 +61,7 @@ "bond-miimon": "mii-monitor-interval", "bond-min-links": "min-links", "bond-mode": "mode", - "bond-num-grat-arp": "gratuitious-arp", + "bond-num-grat-arp": "gratuitous-arp", "bond-primary": "primary", "bond-primary-reselect": "primary-reselect-policy", "bond-updelay": "up-delay", @@ -134,14 +139,16 @@ def __new__(cls, name, parents, dct): class NetworkState(object): - def __init__(self, network_state, version=NETWORK_STATE_VERSION): + def __init__( + self, network_state: dict, version: int = NETWORK_STATE_VERSION + ): self._network_state = copy.deepcopy(network_state) self._version = version self.use_ipv6 = network_state.get("use_ipv6", False) self._has_default_route = None @property - def config(self): + def config(self) -> dict: return self._network_state["config"] @property @@ -202,6 +209,20 @@ def _is_default_route(self, route): route.get("prefix") == 0 and route.get("network") in default_nets ) + @classmethod + def to_passthrough(cls, network_state: dict) -> "NetworkState": + """Instantiates a `NetworkState` without interpreting its data. + + That means only `config` and `version` are copied. + + :param network_state: Network state data. + :return: Instance of `NetworkState`. + """ + kwargs = {} + if "version" in network_state: + kwargs["version"] = network_state["version"] + return cls({"config": network_state}, **kwargs) + class NetworkStateInterpreter(metaclass=CommandHandlerMeta): @@ -216,16 +237,27 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): "config": None, } - def __init__(self, version=NETWORK_STATE_VERSION, config=None): + def __init__( + self, + version=NETWORK_STATE_VERSION, + config=None, + renderer=None, # type: Optional[Renderer] + ): self._version = version self._config = config self._network_state = copy.deepcopy(self.initial_network_state) self._network_state["config"] = config self._parsed = False - self._interface_dns_map = {} + self._interface_dns_map: dict = {} + self._renderer = renderer @property - def network_state(self): + def network_state(self) -> NetworkState: + from cloudinit.net.netplan import Renderer as NetplanRenderer + + if self._version == 2 and isinstance(self._renderer, NetplanRenderer): + LOG.debug("Passthrough netplan v2 config") + return NetworkState.to_passthrough(self._config) return NetworkState(self._network_state, version=self._version) @property @@ -266,10 +298,6 @@ def dump_network_state(self): def as_dict(self): return {"version": self._version, "config": self._config} - def get_network_state(self): - ns = self.network_state - return ns - def parse_config(self, skip_broken=True): if self._version == 1: self.parse_config_v1(skip_broken=skip_broken) @@ -314,6 +342,12 @@ def parse_config_v1(self, skip_broken=True): } def parse_config_v2(self, skip_broken=True): + from cloudinit.net.netplan import Renderer as NetplanRenderer + + if isinstance(self._renderer, NetplanRenderer): + # Nothing to parse as we are going to perform a Netplan passthrough + return + for command_type, command in self._config.items(): if command_type in ["version", "renderer"]: continue @@ -699,15 +733,14 @@ def handle_ethernets(self, command): # * interface name looked up by mac # * value of "eth" key from this loop name = eth - set_name = cfg.get("set-name", None) + set_name = cfg.get("set-name") if set_name: name = set_name elif mac_address and ifaces_by_mac: lcase_mac_address = mac_address.lower() - for iface_mac, iface_name in ifaces_by_mac.items(): - if lcase_mac_address == iface_mac.lower(): - name = iface_name - break + mac = find_interface_name_from_mac(lcase_mac_address) + if mac: + name = mac phy_cmd["name"] = name driver = match.get("driver", None) @@ -763,7 +796,7 @@ def handle_wifis(self, command): " netplan rendering support." ) - def _v2_common(self, cfg): + def _v2_common(self, cfg) -> None: LOG.debug("v2_common: handling config:\n%s", cfg) for iface, dev_cfg in cfg.items(): if "set-name" in dev_cfg: @@ -779,6 +812,15 @@ def _v2_common(self, cfg): if len(dns) > 0: name_cmd.update({"address": dns}) self.handle_nameserver(name_cmd) + + mac_address: Optional[str] = dev_cfg.get("match", {}).get( + "macaddress" + ) + if mac_address: + real_if_name = find_interface_name_from_mac(mac_address) + if real_if_name: + iface = real_if_name + self._handle_individual_nameserver(name_cmd, iface) def _handle_bond_bridge(self, command, cmd_type=None): @@ -795,13 +837,12 @@ def _handle_bond_bridge(self, command, cmd_type=None): for (key, value) in item_cfg.items() if key not in NETWORK_V2_KEY_FILTER ) - # we accept the fixed spelling, but write the old for compatibility - # Xenial does not have an updated netplan which supports the - # correct spelling. LP: #1756701 + # We accept both spellings (as netplan does). LP: #1756701 + # Normalize internally to the new spelling: params = item_params.get("parameters", {}) - grat_value = params.pop("gratuitous-arp", None) + grat_value = params.pop("gratuitious-arp", None) if grat_value: - params["gratuitious-arp"] = grat_value + params["gratuitous-arp"] = grat_value v1_cmd = { "type": cmd_type, @@ -1038,7 +1079,11 @@ def _normalize_subnets(subnets): return [_normalize_subnet(s) for s in subnets] -def parse_net_config_data(net_config, skip_broken=True) -> NetworkState: +def parse_net_config_data( + net_config: dict, + skip_broken: bool = True, + renderer=None, # type: Optional[Renderer] +) -> NetworkState: """Parses the config, returns NetworkState object :param net_config: curtin network config dict @@ -1052,9 +1097,11 @@ def parse_net_config_data(net_config, skip_broken=True) -> NetworkState: config = net_config if version and config is not None: - nsi = NetworkStateInterpreter(version=version, config=config) + nsi = NetworkStateInterpreter( + version=version, config=config, renderer=renderer + ) nsi.parse_config(skip_broken=skip_broken) - state = nsi.get_network_state() + state = nsi.network_state if not state: raise RuntimeError( diff --git a/cloudinit/net/networkd.py b/cloudinit/net/networkd.py index 3bbeb2841..e0a5d8481 100644 --- a/cloudinit/net/networkd.py +++ b/cloudinit/net/networkd.py @@ -7,11 +7,12 @@ # # This file is part of cloud-init. See LICENSE file for license information. -import os from collections import OrderedDict +from typing import Optional from cloudinit import log as logging from cloudinit import subp, util +from cloudinit.net.network_state import NetworkState from . import renderer @@ -45,10 +46,16 @@ def get_final_conf(self): for k, v in sorted(self.conf_dict.items()): if not v: continue - contents += "[" + k + "]\n" - for e in sorted(v): - contents += e + "\n" - contents += "\n" + if k == "Address": + for e in sorted(v): + contents += "[" + k + "]\n" + contents += e + "\n" + contents += "\n" + else: + contents += "[" + k + "]\n" + for e in sorted(v): + contents += e + "\n" + contents += "\n" return contents @@ -218,16 +225,21 @@ def create_network_file(self, link, conf, nwk_dir): util.write_file(net_fn, conf) util.chownbyname(net_fn, net_fn_owner, net_fn_owner) - def render_network_state(self, network_state, templates=None, target=None): - fp_nwkd = self.network_conf_dir + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: + network_dir = self.network_conf_dir if target: - fp_nwkd = subp.target_path(target) + fp_nwkd + network_dir = subp.target_path(target) + network_dir - util.ensure_dir(os.path.dirname(fp_nwkd)) + util.ensure_dir(network_dir) ret_dict = self._render_content(network_state) for k, v in ret_dict.items(): - self.create_network_file(k, v, fp_nwkd) + self.create_network_file(k, v, network_dir) def _render_content(self, ns): ret_dict = {} @@ -243,7 +255,7 @@ def _render_content(self, ns): self.parse_routes(route, cfg) if ns.version == 2: - name = iface["name"] + name: Optional[str] = iface["name"] # network state doesn't give dhcp domain info # using ns.config as a workaround here @@ -258,8 +270,8 @@ def _render_content(self, ns): if dev_cfg.get("set-name") == name: name = dev_name break - - self.dhcp_domain(ns.config["ethernets"][name], cfg) + if name in ns.config["ethernets"]: + self.dhcp_domain(ns.config["ethernets"][name], cfg) ret_dict.update({link: cfg.get_final_conf()}) diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py index da154731c..d7bc19b1c 100644 --- a/cloudinit/net/renderer.py +++ b/cloudinit/net/renderer.py @@ -7,6 +7,7 @@ import abc import io +from typing import Optional from cloudinit.net.network_state import NetworkState, parse_net_config_data from cloudinit.net.udev import generate_udev_rule @@ -49,11 +50,19 @@ def _render_persistent_net(network_state: NetworkState): return content.getvalue() @abc.abstractmethod - def render_network_state(self, network_state, templates=None, target=None): + def render_network_state( + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: """Render network state.""" def render_network_config( - self, network_config, templates=None, target=None + self, + network_config: dict, + templates: Optional[dict] = None, + target=None, ): return self.render_network_state( network_state=parse_net_config_data(network_config), diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 37c5d260b..d5789fb00 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -4,7 +4,7 @@ import io import os import re -from typing import Mapping +from typing import Mapping, Optional from cloudinit import log as logging from cloudinit import subp, util @@ -28,6 +28,7 @@ "fedora", "miraclelinux", "openEuler", + "openmandriva", "rhel", "rocky", "suse", @@ -361,7 +362,7 @@ class Renderer(renderer.Renderer): ] ) - templates = {} + templates: dict = {} def __init__(self, config=None): if not config: @@ -979,8 +980,11 @@ def _render_sysconfig( return contents def render_network_state( - self, network_state: NetworkState, templates=None, target=None - ): + self, + network_state: NetworkState, + templates: Optional[dict] = None, + target=None, + ) -> None: if not templates: templates = self.templates file_mode = 0o644 diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index 06b5b49fa..b839eaae1 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -9,8 +9,10 @@ report events in a structured manner. """ -from ..registry import DictRegistry -from .handlers import available_handlers +from typing import Type + +from cloudinit.registry import DictRegistry +from cloudinit.reporting.handlers import HandlerType, available_handlers DEFAULT_CONFIG = { "logging": {"type": "log"}, @@ -32,19 +34,19 @@ def update_configuration(config): ) continue handler_config = handler_config.copy() - cls = available_handlers.registered_items[handler_config.pop("type")] + cls: Type[HandlerType] = available_handlers.registered_items[ + handler_config.pop("type") + ] instantiated_handler_registry.unregister_item(handler_name) - instance = cls(**handler_config) + instance = cls(**handler_config) # pyright: ignore instantiated_handler_registry.register_item(handler_name, instance) def flush_events(): - for _, handler in instantiated_handler_registry.registered_items.items(): - if hasattr(handler, "flush"): - handler.flush() + handler: HandlerType + for handler in instantiated_handler_registry.registered_items.values(): + handler.flush() instantiated_handler_registry = DictRegistry() update_configuration(DEFAULT_CONFIG) - -# vi: ts=4 expandtab diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py index e53186a39..34c3b8756 100644 --- a/cloudinit/reporting/events.py +++ b/cloudinit/reporting/events.py @@ -11,6 +11,9 @@ import base64 import os.path import time +from typing import List + +from cloudinit.reporting.handlers import ReportingHandler from . import available_handlers, instantiated_handler_registry @@ -116,8 +119,10 @@ def report_event(event, excluded_handler_types=None): if hndl_type in excluded_handler_types } - handlers = instantiated_handler_registry.registered_items.items() - for _, handler in handlers: + handlers: List[ReportingHandler] = list( + instantiated_handler_registry.registered_items.values() + ) + for handler in handlers: if type(handler) in excluded_handler_classes: continue # skip this excluded handler handler.publish_event(event) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index e163e168f..d43b80b0d 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -10,6 +10,8 @@ import time import uuid from datetime import datetime +from threading import Event +from typing import Union from cloudinit import log as logging from cloudinit import url_helper, util @@ -81,34 +83,79 @@ def __init__( super(WebHookHandler, self).__init__() if any([consumer_key, token_key, token_secret, consumer_secret]): - self.oauth_helper = url_helper.OauthUrlHelper( + oauth_helper = url_helper.OauthUrlHelper( consumer_key=consumer_key, token_key=token_key, token_secret=token_secret, consumer_secret=consumer_secret, ) + self.readurl = oauth_helper.readurl else: - self.oauth_helper = None + self.readurl = url_helper.readurl self.endpoint = endpoint self.timeout = timeout self.retries = retries self.ssl_details = util.fetch_ssl_details() + self.flush_requested = Event() + self.queue = queue.Queue() + self.event_processor = threading.Thread(target=self.process_requests) + self.event_processor.daemon = True + self.event_processor.start() + + def process_requests(self): + consecutive_failed = 0 + while True: + if self.flush_requested.is_set() and consecutive_failed > 2: + # At this point the main thread is waiting for the queue to + # drain. If we have a queue of events piled up and recent + # events have failed, lets not waste time trying to post + # the rest, especially since a long timeout could block + # cloud-init for quite a long time. + LOG.warning( + "Multiple consecutive failures in WebHookHandler. " + "Cancelling all queued events." + ) + while not self.queue.empty(): + self.queue.get_nowait() + self.queue.task_done() + consecutive_failed = 0 + args = self.queue.get(block=True) + try: + self.readurl( + args[0], + data=args[1], + timeout=args[2], + retries=args[3], + ssl_details=args[4], + ) + consecutive_failed = 0 + except Exception as e: + LOG.warning( + "Failed posting event: %s. This was caused by: %s", + args[1], + e, + ) + consecutive_failed += 1 + finally: + self.queue.task_done() + def publish_event(self, event): - if self.oauth_helper: - readurl = self.oauth_helper.readurl - else: - readurl = url_helper.readurl - try: - return readurl( + self.queue.put( + ( self.endpoint, - data=json.dumps(event.as_dict()), - timeout=self.timeout, - retries=self.retries, - ssl_details=self.ssl_details, + json.dumps(event.as_dict()), + self.timeout, + self.retries, + self.ssl_details, ) - except Exception: - LOG.warning("failed posting event: %s", event.as_string()) + ) + + def flush(self): + self.flush_requested.set() + LOG.debug("WebHookHandler flushing remaining events") + self.queue.join() + self.flush_requested.clear() class HyperVKvpReportingHandler(ReportingHandler): @@ -359,10 +406,18 @@ def flush(self): self.q.join() +# Type[ReportingHandler] doesn't work here because each class has different +# call args. Protocols in python 3.8 can probably make this simpler. +HandlerType = Union[ + ReportingHandler, + LogHandler, + PrintHandler, + WebHookHandler, + HyperVKvpReportingHandler, +] + available_handlers = DictRegistry() available_handlers.register_item("log", LogHandler) available_handlers.register_item("print", PrintHandler) available_handlers.register_item("webhook", WebHookHandler) available_handlers.register_item("hyperv", HyperVKvpReportingHandler) - -# vi: ts=4 expandtab diff --git a/cloudinit/safeyaml.py b/cloudinit/safeyaml.py index eeb6f82b3..368ac861c 100644 --- a/cloudinit/safeyaml.py +++ b/cloudinit/safeyaml.py @@ -57,7 +57,7 @@ class _CustomSafeLoaderWithMarks(yaml.SafeLoader): def __init__(self, stream): super().__init__(stream) - self.schemamarks_by_line = {} # type: Dict[int, List[SchemaPathMarks]] + self.schemamarks_by_line: Dict[int, List[SchemaPathMarks]] = {} def _get_nested_path_prefix(self, node): if node.start_mark.line in self.schemamarks_by_line: diff --git a/cloudinit/serial.py b/cloudinit/serial.py deleted file mode 100644 index a6f710ef3..000000000 --- a/cloudinit/serial.py +++ /dev/null @@ -1,46 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -try: - from serial import Serial -except ImportError: - # For older versions of python (ie 2.6) pyserial may not exist and/or - # work and/or be installed, so make a dummy/fake serial that blows up - # when used... - class Serial(object): - def __init__(self, *args, **kwargs): - pass - - @staticmethod - def isOpen(): - return False - - @staticmethod - def write(data): - raise IOError( - "Unable to perform serial `write` operation," - " pyserial not installed." - ) - - @staticmethod - def readline(): - raise IOError( - "Unable to perform serial `readline` operation," - " pyserial not installed." - ) - - @staticmethod - def flush(): - raise IOError( - "Unable to perform serial `flush` operation," - " pyserial not installed." - ) - - @staticmethod - def read(size=1): - raise IOError( - "Unable to perform serial `read` operation," - " pyserial not installed." - ) - - -# vi: ts=4 expandtab diff --git a/cloudinit/settings.py b/cloudinit/settings.py index ecc1403bd..32844e71d 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -14,6 +14,8 @@ # This is expected to be a yaml formatted file CLOUD_CONFIG = "/etc/cloud/cloud.cfg" +CLEAN_RUNPARTS_DIR = "/etc/cloud/clean.d" + RUN_CLOUD_CONFIG = "/run/cloud-init/cloud.cfg" # What u get if no config is provided diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index 37f512e3e..6804274e1 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -1,7 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +from typing import List + from cloudinit import dmi, sources from cloudinit.sources import DataSourceEc2 as EC2 +from cloudinit.sources import DataSourceHostname ALIYUN_PRODUCT = "Alibaba Cloud ECS" @@ -13,10 +16,15 @@ class DataSourceAliYun(EC2.DataSourceEc2): # The minimum supported metadata_version from the ec2 metadata apis min_metadata_version = "2016-01-01" - extended_metadata_versions = [] + extended_metadata_versions: List[str] = [] def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): - return self.metadata.get("hostname", "localhost.localdomain") + hostname = self.metadata.get("hostname") + is_default = False + if hostname is None: + hostname = "localhost.localdomain" + is_default = True + return DataSourceHostname(hostname, is_default) def get_public_ssh_keys(self): return parse_public_keys(self.metadata.get("public-keys", {})) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bfb404960..354949126 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -12,9 +12,9 @@ import re import xml.etree.ElementTree as ET from enum import Enum +from pathlib import Path from time import sleep, time from typing import Any, Dict, List, Optional -from xml.dom import minidom import requests @@ -24,16 +24,20 @@ from cloudinit.event import EventScope, EventType from cloudinit.net import device_driver from cloudinit.net.dhcp import ( - EphemeralDHCPv4, NoDHCPLeaseError, NoDHCPLeaseInterfaceError, NoDHCPLeaseMissingDhclientError, ) +from cloudinit.net.ephemeral import EphemeralDHCPv4 from cloudinit.reporting import events from cloudinit.sources.helpers import netlink from cloudinit.sources.helpers.azure import ( DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE, DEFAULT_WIRESERVER_ENDPOINT, + BrokenAzureDataSource, + ChassisAssetTag, + NonAzureDataSource, + OvfEnvXml, azure_ds_reporter, azure_ds_telemetry_reporter, build_minimal_ovf, @@ -58,9 +62,6 @@ # ensures that it gets linked to this path. RESOURCE_DISK_PATH = "/dev/disk/cloud/azure_resource" DEFAULT_FS = "ext4" -# DMI chassis-asset-tag is set static for all azure instances -AZURE_CHASSIS_ASSET_TAG = "7783-7084-3265-9085-8269-3286-77" -REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready" AGENT_SEED_DIR = "/var/lib/waagent" DEFAULT_PROVISIONING_ISO_DEV = "/dev/sr0" @@ -92,6 +93,7 @@ class MetadataType(Enum): class PPSType(Enum): NONE = "None" + OS_DISK = "PreprovisionedOSDisk" RUNNING = "Running" SAVABLE = "Savable" UNKNOWN = "Unknown" @@ -100,7 +102,7 @@ class PPSType(Enum): PLATFORM_ENTROPY_SOURCE: Optional[str] = "/sys/firmware/acpi/tables/OEM0" # List of static scripts and network config artifacts created by -# stock ubuntu suported images. +# stock ubuntu supported images. UBUNTU_EXTENDED_NETWORK_SCRIPTS = [ "/etc/netplan/90-hotplug-azure.yaml", "/usr/local/sbin/ephemeral_eth.sh", @@ -208,7 +210,7 @@ def get_hv_netvsc_macs_normalized() -> List[str]: def execute_or_debug(cmd, fail_ret=None) -> str: try: - return subp.subp(cmd).stdout # type: ignore + return subp.subp(cmd).stdout # pyright: ignore except subp.ProcessExecutionError: LOG.debug("Failed to execute: %s", " ".join(cmd)) return fail_ret @@ -285,7 +287,6 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, "apply_network_config": True, # Use IMDS published network configuration } -# RELEASE_BLOCKER: Xenial and earlier apply_network_config default is False BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG = { "disk_setup": { @@ -307,6 +308,20 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: DEF_PASSWD_REDACTION = "REDACTED" +@azure_ds_telemetry_reporter +def is_platform_viable(seed_dir: Optional[Path]) -> bool: + """Check platform environment to report if this datasource may run.""" + chassis_tag = ChassisAssetTag.query_system() + if chassis_tag is not None: + return True + + # If no valid chassis tag, check for seeded ovf-env.xml. + if seed_dir is None: + return False + + return (seed_dir / "ovf-env.xml").exists() + + class DataSourceAzure(sources.DataSource): dsname = "Azure" @@ -332,6 +347,9 @@ def __init__(self, sys_cfg, distro, paths): self._network_config = None self._ephemeral_dhcp_ctx = None self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT + self._reported_ready_marker_file = os.path.join( + paths.cloud_dir, "data", "reported_ready" + ) def _unpickle(self, ci_pkl_version: int) -> None: super()._unpickle(ci_pkl_version) @@ -339,6 +357,9 @@ def _unpickle(self, ci_pkl_version: int) -> None: self._ephemeral_dhcp_ctx = None self._iso_dev = None self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT + self._reported_ready_marker_file = os.path.join( + self.paths.cloud_dir, "data", "reported_ready" + ) def __str__(self): root = sources.DataSource.__str__(self) @@ -560,6 +581,9 @@ def crawl_metadata(self): if pps_type == PPSType.SAVABLE: self._wait_for_all_nics_ready() + elif pps_type == PPSType.OS_DISK: + self._report_ready_for_pps(create_marker=False) + self._wait_for_pps_os_disk_shutdown() md, userdata_raw, cfg, files = self._reprovision() # fetch metadata again as it has changed after reprovisioning @@ -598,9 +622,9 @@ def crawl_metadata(self): if metadata_source == "IMDS" and not crawled_data["files"]: try: contents = build_minimal_ovf( - username=imds_username, # type: ignore - hostname=imds_hostname, # type: ignore - disableSshPwd=imds_disable_password, # type: ignore + username=imds_username, # pyright: ignore + hostname=imds_hostname, # pyright: ignore + disableSshPwd=imds_disable_password, # pyright: ignore ) crawled_data["files"] = {"ovf-env.xml": contents} except Exception as e: @@ -664,10 +688,6 @@ def crawl_metadata(self): return crawled_data - def _is_platform_viable(self): - """Check platform environment to report if this datasource may run.""" - return _is_platform_viable(self.seed_dir) - def clear_cached_attrs(self, attr_defaults=()): """Reset any cached class attributes to defaults.""" super(DataSourceAzure, self).clear_cached_attrs(attr_defaults) @@ -680,7 +700,7 @@ def _get_data(self): @return: True on success, False on error, invalid or disabled datasource. """ - if not self._is_platform_viable(): + if not is_platform_viable(Path(self.seed_dir)): return False try: get_boot_telemetry() @@ -748,9 +768,6 @@ def _get_data(self): ) self.userdata_raw = crawled_data["userdata_raw"] - user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) - self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) - # walinux agent writes files world readable, but expects # the directory to be protected. write_files( @@ -806,6 +823,11 @@ def get_imds_data_with_api_fallback( ) return {} + def get_instance_id(self): + if not self.metadata or "instance-id" not in self.metadata: + return self._iid() + return str(self.metadata["instance-id"]) + def device_name_to_device(self, name): return self.ds_cfg["disk_aliases"].get(name) @@ -948,7 +970,7 @@ def wait_for_link_up( @azure_ds_telemetry_reporter def _create_report_ready_marker(self): - path = REPORTED_READY_MARKER_FILE + path = self._reported_ready_marker_file LOG.info("Creating a marker file to report ready: %s", path) util.write_file( path, "{pid}: {time}\n".format(pid=os.getpid(), time=time()) @@ -960,7 +982,12 @@ def _create_report_ready_marker(self): ) @azure_ds_telemetry_reporter - def _report_ready_for_pps(self) -> None: + def _report_ready_for_pps( + self, + *, + create_marker: bool = True, + expect_url_error: bool = False, + ) -> None: """Report ready for PPS, creating the marker file upon completion. :raises sources.InvalidMetaDataException: On error reporting ready. @@ -968,11 +995,36 @@ def _report_ready_for_pps(self) -> None: try: self._report_ready() except Exception as error: - msg = "Failed reporting ready while in the preprovisioning pool." - report_diagnostic_event(msg, logger_func=LOG.error) - raise sources.InvalidMetaDataException(msg) from error + # Ignore HTTP failures for Savable PPS as the call may appear to + # fail if the network interface is unplugged or the VM is + # suspended before we process the response. Worst case scenario + # is that we failed to report ready for source PPS and this VM + # will be discarded shortly, no harm done. + if expect_url_error and isinstance(error, UrlError): + report_diagnostic_event( + "Ignoring http call failure, it was expected.", + logger_func=LOG.debug, + ) + # The iso was ejected prior to reporting ready. + self._iso_dev = None + else: + msg = ( + "Failed reporting ready while in the preprovisioning pool." + ) + report_diagnostic_event(msg, logger_func=LOG.error) + raise sources.InvalidMetaDataException(msg) from error - self._create_report_ready_marker() + if create_marker: + self._create_report_ready_marker() + + @azure_ds_telemetry_reporter + def _wait_for_pps_os_disk_shutdown(self): + report_diagnostic_event( + "Waiting for host to shutdown VM...", + logger_func=LOG.info, + ) + sleep(31536000) + raise BrokenAzureDataSource("Shutdown failure for PPS disk.") @azure_ds_telemetry_reporter def _check_if_nic_is_primary(self, ifname): @@ -1133,8 +1185,17 @@ def _wait_for_all_nics_ready(self): nl_sock = None try: nl_sock = netlink.create_bound_netlink_socket() - self._report_ready_for_pps() - self._teardown_ephemeral_networking() + self._report_ready_for_pps(expect_url_error=True) + try: + self._teardown_ephemeral_networking() + except subp.ProcessExecutionError as e: + report_diagnostic_event( + "Ignoring failure while tearing down networking, " + "NIC was likely unplugged: %r" % e, + logger_func=LOG.info, + ) + self._ephemeral_dhcp_ctx = None + self._wait_for_nic_detach(nl_sock) self._wait_for_hot_attached_primary_nic(nl_sock) except netlink.NetlinkCreateSocketError as e: @@ -1153,7 +1214,9 @@ def _poll_imds(self): ) headers = {"Metadata": "true"} nl_sock = None - report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE)) + report_ready = bool( + not os.path.isfile(self._reported_ready_marker_file) + ) self.imds_logging_threshold = 1 self.imds_poll_counter = 1 dhcp_attempts = 0 @@ -1383,13 +1446,18 @@ def _ppstype_from_imds(self, imds_md: dict) -> Optional[str]: def _determine_pps_type(self, ovf_cfg: dict, imds_md: dict) -> PPSType: """Determine PPS type using OVF, IMDS data, and reprovision marker.""" - if os.path.isfile(REPORTED_READY_MARKER_FILE): + if os.path.isfile(self._reported_ready_marker_file): pps_type = PPSType.UNKNOWN elif ( ovf_cfg.get("PreprovisionedVMType", None) == PPSType.SAVABLE.value or self._ppstype_from_imds(imds_md) == PPSType.SAVABLE.value ): pps_type = PPSType.SAVABLE + elif ( + ovf_cfg.get("PreprovisionedVMType", None) == PPSType.OS_DISK.value + or self._ppstype_from_imds(imds_md) == PPSType.OS_DISK.value + ): + pps_type = PPSType.OS_DISK elif ( ovf_cfg.get("PreprovisionedVm") is True or ovf_cfg.get("PreprovisionedVMType", None) @@ -1441,12 +1509,14 @@ def _determine_wireserver_pubkey_info( def _cleanup_markers(self): """Cleanup any marker files.""" - util.del_file(REPORTED_READY_MARKER_FILE) + util.del_file(self._reported_ready_marker_file) @azure_ds_telemetry_reporter def activate(self, cfg, is_new_instance): + instance_dir = self.paths.get_ipath_cur() try: address_ephemeral_resize( + instance_dir, is_new_instance=is_new_instance, preserve_ntfs=self.ds_cfg.get(DS_CFG_KEY_PRESERVE_NTFS, False), ) @@ -1462,22 +1532,42 @@ def availability_zone(self): .get("platformFaultDomain") ) + @azure_ds_telemetry_reporter + def _generate_network_config(self): + """Generate network configuration according to configuration.""" + # Use IMDS network metadata, if configured. + if ( + self._metadata_imds + and self._metadata_imds != sources.UNSET + and self.ds_cfg.get("apply_network_config") + ): + try: + return generate_network_config_from_instance_network_metadata( + self._metadata_imds["network"] + ) + except Exception as e: + LOG.error( + "Failed generating network config " + "from IMDS network metadata: %s", + str(e), + ) + + # Generate fallback configuration. + try: + return _generate_network_config_from_fallback_config() + except Exception as e: + LOG.error("Failed generating fallback network config: %s", str(e)) + + return {} + @property def network_config(self): - """Generate a network config like net.generate_fallback_network() with - the following exceptions. + """Provide network configuration v2 dictionary.""" + # Use cached config, if present. + if self._network_config and self._network_config != sources.UNSET: + return self._network_config - 1. Probe the drivers of the net-devices present and inject them in - the network configuration under params: driver: value - 2. Generate a fallback network config that does not include any of - the blacklisted devices. - """ - if not self._network_config or self._network_config == sources.UNSET: - if self.ds_cfg.get("apply_network_config"): - nc_src = self._metadata_imds - else: - nc_src = None - self._network_config = parse_network_config(nc_src) + self._network_config = self._generate_network_config() return self._network_config @property @@ -1710,7 +1800,10 @@ def count_files(mp): @azure_ds_telemetry_reporter def address_ephemeral_resize( - devpath=RESOURCE_DISK_PATH, is_new_instance=False, preserve_ntfs=False + instance_dir: str, + devpath: str = RESOURCE_DISK_PATH, + is_new_instance: bool = False, + preserve_ntfs: bool = False, ): if not os.path.exists(devpath): report_diagnostic_event( @@ -1736,14 +1829,13 @@ def address_ephemeral_resize( return for mod in ["disk_setup", "mounts"]: - sempath = "/var/lib/cloud/instance/sem/config_" + mod + sempath = os.path.join(instance_dir, "sem", "config_" + mod) bmsg = 'Marker "%s" for module "%s"' % (sempath, mod) if os.path.exists(sempath): try: os.unlink(sempath) LOG.debug("%s removed.", bmsg) - except Exception as e: - # python3 throws FileNotFoundError, python2 throws OSError + except FileNotFoundError as e: LOG.warning("%s: remove failed! (%s)", bmsg, e) else: LOG.debug("%s did not exist.", bmsg) @@ -1779,285 +1871,54 @@ def _redact_password(cnt, fname): util.write_file(filename=fname, content=content, mode=0o600) -def find_child(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return ret - - -@azure_ds_telemetry_reporter -def load_azure_ovf_pubkeys(sshnode): - # This parses a 'SSH' node formatted like below, and returns - # an array of dicts. - # [{'fingerprint': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', - # 'path': '/where/to/go'}] - # - # - # ABC/x/y/z - # ... - # - # Under some circumstances, there may be a element along with the - # Fingerprint and Path. Pass those along if they appear. - results = find_child(sshnode, lambda n: n.localName == "PublicKeys") - if len(results) == 0: - return [] - if len(results) > 1: - raise BrokenAzureDataSource( - "Multiple 'PublicKeys'(%s) in SSH node" % len(results) - ) - - pubkeys_node = results[0] - pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey") - - if len(pubkeys) == 0: - return [] - - found = [] - text_node = minidom.Document.TEXT_NODE - - for pk_node in pubkeys: - if not pk_node.hasChildNodes(): - continue - - cur = {"fingerprint": "", "path": "", "value": ""} - for child in pk_node.childNodes: - if child.nodeType == text_node or not child.localName: - continue - - name = child.localName.lower() - - if name not in cur.keys(): - continue - - if ( - len(child.childNodes) != 1 - or child.childNodes[0].nodeType != text_node - ): - continue - - cur[name] = child.childNodes[0].wholeText.strip() - found.append(cur) - - return found - - @azure_ds_telemetry_reporter def read_azure_ovf(contents): - try: - dom = minidom.parseString(contents) - except Exception as e: - error_str = "Invalid ovf-env.xml: %s" % e - report_diagnostic_event(error_str, logger_func=LOG.warning) - raise BrokenAzureDataSource(error_str) from e - - results = find_child( - dom.documentElement, lambda n: n.localName == "ProvisioningSection" - ) - - if len(results) == 0: - raise NonAzureDataSource("No ProvisioningSection") - if len(results) > 1: - raise BrokenAzureDataSource( - "found '%d' ProvisioningSection items" % len(results) - ) - provSection = results[0] - - lpcs_nodes = find_child( - provSection, - lambda n: n.localName == "LinuxProvisioningConfigurationSet", - ) - - if len(lpcs_nodes) == 0: - raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") - if len(lpcs_nodes) > 1: - raise BrokenAzureDataSource( - "found '%d' %ss" - % (len(lpcs_nodes), "LinuxProvisioningConfigurationSet") - ) - lpcs = lpcs_nodes[0] + """Parse OVF XML contents. - if not lpcs.hasChildNodes(): - raise BrokenAzureDataSource("no child nodes of configuration set") + :return: Tuple of metadata, configuration, userdata dicts. - md_props = "seedfrom" - md: Dict[str, Any] = {"azure_data": {}} + :raises NonAzureDataSource: if XML is not in Azure's format. + :raises BrokenAzureDataSource: if XML is unparseable or invalid. + """ + ovf_env = OvfEnvXml.parse_text(contents) + md: Dict[str, Any] = {} cfg = {} - ud = "" - password = None - username = None - - for child in lpcs.childNodes: - if child.nodeType == dom.TEXT_NODE or not child.localName: - continue + ud = ovf_env.custom_data or "" - name = child.localName.lower() + if ovf_env.hostname: + md["local-hostname"] = ovf_env.hostname - simple = False - value = "" - if ( - len(child.childNodes) == 1 - and child.childNodes[0].nodeType == dom.TEXT_NODE - ): - simple = True - value = child.childNodes[0].wholeText + if ovf_env.public_keys: + cfg["_pubkeys"] = ovf_env.public_keys - attrs = dict([(k, v) for k, v in child.attributes.items()]) - - # we accept either UserData or CustomData. If both are present - # then behavior is undefined. - if name == "userdata" or name == "customdata": - if attrs.get("encoding") in (None, "base64"): - ud = base64.b64decode("".join(value.split())) - else: - ud = value - elif name == "username": - username = value - elif name == "userpassword": - password = value - elif name == "hostname": - md["local-hostname"] = value - elif name == "dscfg": - if attrs.get("encoding") in (None, "base64"): - dscfg = base64.b64decode("".join(value.split())) - else: - dscfg = value - cfg["datasource"] = {DS_NAME: util.load_yaml(dscfg, default={})} - elif name == "ssh": - cfg["_pubkeys"] = load_azure_ovf_pubkeys(child) - elif name == "disablesshpasswordauthentication": - cfg["ssh_pwauth"] = util.is_false(value) - elif simple: - if name in md_props: - md[name] = value - else: - md["azure_data"][name] = value + if ovf_env.disable_ssh_password_auth is not None: + cfg["ssh_pwauth"] = not ovf_env.disable_ssh_password_auth + elif ovf_env.password: + cfg["ssh_pwauth"] = True defuser = {} - if username: - defuser["name"] = username - if password: + if ovf_env.username: + defuser["name"] = ovf_env.username + if ovf_env.password: defuser["lock_passwd"] = False - if DEF_PASSWD_REDACTION != password: - defuser["passwd"] = cfg["password"] = encrypt_pass(password) + if DEF_PASSWD_REDACTION != ovf_env.password: + defuser["hashed_passwd"] = encrypt_pass(ovf_env.password) if defuser: cfg["system_info"] = {"default_user": defuser} - if "ssh_pwauth" not in cfg and password: - cfg["ssh_pwauth"] = True - - preprovisioning_cfg = _get_preprovisioning_cfgs(dom) - cfg = util.mergemanydict([cfg, preprovisioning_cfg]) - - return (md, ud, cfg) - - -@azure_ds_telemetry_reporter -def _get_preprovisioning_cfgs(dom): - """Read the preprovisioning related flags from ovf and populates a dict - with the info. - - Two flags are in use today: PreprovisionedVm bool and - PreprovisionedVMType enum. In the long term, the PreprovisionedVm bool - will be deprecated in favor of PreprovisionedVMType string/enum. - - Only these combinations of values are possible today: - - PreprovisionedVm=True and PreprovisionedVMType=Running - - PreprovisionedVm=False and PreprovisionedVMType=Savable - - PreprovisionedVm is missing and PreprovisionedVMType=Running/Savable - - PreprovisionedVm=False and PreprovisionedVMType is missing - - More specifically, this will never happen: - - PreprovisionedVm=True and PreprovisionedVMType=Savable - """ - cfg = {"PreprovisionedVm": False, "PreprovisionedVMType": None} - - platform_settings_section = find_child( - dom.documentElement, lambda n: n.localName == "PlatformSettingsSection" - ) - if not platform_settings_section or len(platform_settings_section) == 0: - LOG.debug("PlatformSettingsSection not found") - return cfg - platform_settings = find_child( - platform_settings_section[0], - lambda n: n.localName == "PlatformSettings", - ) - if not platform_settings or len(platform_settings) == 0: - LOG.debug("PlatformSettings not found") - return cfg - - # Read the PreprovisionedVm bool flag. This should be deprecated when the - # platform has removed PreprovisionedVm and only surfaces - # PreprovisionedVMType. - cfg["PreprovisionedVm"] = _get_preprovisionedvm_cfg_value( - platform_settings - ) - - cfg["PreprovisionedVMType"] = _get_preprovisionedvmtype_cfg_value( - platform_settings - ) - return cfg - - -@azure_ds_telemetry_reporter -def _get_preprovisionedvm_cfg_value(platform_settings): - preprovisionedVm = False - - # Read the PreprovisionedVm bool flag. This should be deprecated when the - # platform has removed PreprovisionedVm and only surfaces - # PreprovisionedVMType. - preprovisionedVmVal = find_child( - platform_settings[0], lambda n: n.localName == "PreprovisionedVm" - ) - if not preprovisionedVmVal or len(preprovisionedVmVal) == 0: - LOG.debug("PreprovisionedVm not found") - return preprovisionedVm - preprovisionedVm = util.translate_bool( - preprovisionedVmVal[0].firstChild.nodeValue - ) - + cfg["PreprovisionedVm"] = ovf_env.preprovisioned_vm report_diagnostic_event( - "PreprovisionedVm: %s" % preprovisionedVm, logger_func=LOG.info - ) - - return preprovisionedVm - - -@azure_ds_telemetry_reporter -def _get_preprovisionedvmtype_cfg_value(platform_settings): - preprovisionedVMType = None - - # Read the PreprovisionedVMType value from the ovf. It can be - # 'Running' or 'Savable' or not exist. This enum value is intended to - # replace PreprovisionedVm bool flag in the long term. - # A Running VM is the same as preprovisioned VMs of today. This is - # equivalent to having PreprovisionedVm=True. - # A Savable VM is one whose nic is hot-detached immediately after it - # reports ready the first time to free up the network resources. - # Once assigned to customer, the customer-requested nics are - # hot-attached to it and reprovision happens like today. - preprovisionedVMTypeVal = find_child( - platform_settings[0], lambda n: n.localName == "PreprovisionedVMType" + "PreprovisionedVm: %s" % ovf_env.preprovisioned_vm, + logger_func=LOG.info, ) - if ( - not preprovisionedVMTypeVal - or len(preprovisionedVMTypeVal) == 0 - or preprovisionedVMTypeVal[0].firstChild is None - ): - LOG.debug("PreprovisionedVMType not found") - return preprovisionedVMType - - preprovisionedVMType = preprovisionedVMTypeVal[0].firstChild.nodeValue + cfg["PreprovisionedVMType"] = ovf_env.preprovisioned_vm_type report_diagnostic_event( - "PreprovisionedVMType: %s" % preprovisionedVMType, logger_func=LOG.info + "PreprovisionedVMType: %s" % ovf_env.preprovisioned_vm_type, + logger_func=LOG.info, ) - - return preprovisionedVMType + return (md, ud, cfg) def encrypt_pass(password, salt_id="$6$"): @@ -2087,15 +1948,14 @@ def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): seed = util.load_file(source, quiet=True, decode=False) # The seed generally contains non-Unicode characters. load_file puts - # them into a str (in python 2) or bytes (in python 3). In python 2, - # bad octets in a str cause util.json_dumps() to throw an exception. In - # python 3, bytes is a non-serializable type, and the handler load_file + # them into bytes (in python 3). + # bytes is a non-serializable type, and the handler load_file # uses applies b64 encoding *again* to handle it. The simplest solution # is to just b64encode the data and then decode it to a serializable # string. Same number of bits of entropy, just with 25% more zeroes. # There's no need to undo this base64-encoding when the random seed is # actually used in cc_seed_random.py. - return base64.b64encode(seed).decode() # type: ignore + return base64.b64encode(seed).decode() # pyright: ignore @azure_ds_telemetry_reporter @@ -2128,40 +1988,16 @@ def load_azure_ds_dir(source_dir): @azure_ds_telemetry_reporter -def parse_network_config(imds_metadata) -> dict: - """Convert imds_metadata dictionary to network v2 configuration. - Parses network configuration from imds metadata if present or generate - fallback network config excluding mlx4_core devices. - - @param: imds_metadata: Dict of content read from IMDS network service. - @return: Dictionary containing network version 2 standard configuration. - """ - if imds_metadata != sources.UNSET and imds_metadata: - try: - return _generate_network_config_from_imds_metadata(imds_metadata) - except Exception as e: - LOG.error( - "Failed generating network config " - "from IMDS network metadata: %s", - str(e), - ) - try: - return _generate_network_config_from_fallback_config() - except Exception as e: - LOG.error("Failed generating fallback network config: %s", str(e)) - return {} +def generate_network_config_from_instance_network_metadata( + network_metadata: dict, +) -> dict: + """Convert imds network metadata dictionary to network v2 configuration. + :param: network_metadata: Dict of "network" key from instance metdata. -@azure_ds_telemetry_reporter -def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: - """Convert imds_metadata dictionary to network v2 configuration. - Parses network configuration from imds metadata. - - @param: imds_metadata: Dict of content read from IMDS network service. - @return: Dictionary containing network version 2 standard configuration. + :return: Dictionary containing network version 2 standard configuration. """ netconfig: Dict[str, Any] = {"version": 2, "ethernets": {}} - network_metadata = imds_metadata["network"] for idx, intf in enumerate(network_metadata["interface"]): has_ip_address = False # First IPv4 and/or IPv6 address will be obtained via DHCP. @@ -2367,32 +2203,6 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): util.del_file(path) -def _is_platform_viable(seed_dir): - """Check platform environment to report if this datasource may run.""" - with events.ReportEventStack( - name="check-platform-viability", - description="found azure asset tag", - parent=azure_ds_reporter, - ) as evt: - asset_tag = dmi.read_dmi_data("chassis-asset-tag") - if asset_tag == AZURE_CHASSIS_ASSET_TAG: - return True - msg = "Non-Azure DMI asset tag '%s' discovered." % asset_tag - evt.description = msg - report_diagnostic_event(msg, logger_func=LOG.debug) - if os.path.exists(os.path.join(seed_dir, "ovf-env.xml")): - return True - return False - - -class BrokenAzureDataSource(Exception): - pass - - -class NonAzureDataSource(Exception): - pass - - # Legacy: Must be present in case we load an old pkl object DataSourceAzureNet = DataSourceAzure diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index 426a762ec..8d01b5631 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -6,6 +6,7 @@ import errno import json +import os from cloudinit import sources, url_helper, util @@ -15,13 +16,13 @@ class DataSourceBigstep(sources.DataSource): dsname = "Bigstep" def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) + super().__init__(sys_cfg, distro, paths) self.metadata = {} self.vendordata_raw = "" self.userdata_raw = "" - def _get_data(self, apply_filter=False): - url = get_url_from_file() + def _get_data(self, apply_filter=False) -> bool: + url = self._get_url_from_file() if url is None: return False response = url_helper.readurl(url) @@ -31,22 +32,25 @@ def _get_data(self, apply_filter=False): self.userdata_raw = decoded["userdata_raw"] return True - def _get_subplatform(self): + def _get_subplatform(self) -> str: """Return the subplatform metadata source details.""" - return "metadata (%s)" % get_url_from_file() - - -def get_url_from_file(): - try: - content = util.load_file("/var/lib/cloud/data/seed/bigstep/url") - except IOError as e: - # If the file doesn't exist, then the server probably isn't a Bigstep - # instance; otherwise, another problem exists which needs investigation - if e.errno == errno.ENOENT: - return None - else: - raise - return content + return f"metadata ({self._get_url_from_file()})" + + def _get_url_from_file(self): + url_file = os.path.join( + self.paths.cloud_dir, "data", "seed", "bigstep", "url" + ) + try: + content = util.load_file(url_file) + except IOError as e: + # If the file doesn't exist, then the server probably isn't a + # Bigstep instance; otherwise, another problem exists which needs + # investigation + if e.errno == errno.ENOENT: + return None + else: + raise + return content # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 7d7021372..270a3a186 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -10,6 +10,7 @@ from cloudinit import dmi from cloudinit import log as logging from cloudinit import sources +from cloudinit.sources import DataSourceHostname from cloudinit.sources.helpers.cloudsigma import SERIAL_PORT, Cepko LOG = logging.getLogger(__name__) @@ -90,9 +91,10 @@ def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): the first part from uuid is being used. """ if re.match(r"^[A-Za-z0-9 -_\.]+$", self.metadata["name"]): - return self.metadata["name"][:61] + ret = self.metadata["name"][:61] else: - return self.metadata["uuid"].split("-")[0] + ret = self.metadata["uuid"].split("-")[0] + return DataSourceHostname(ret, False) def get_public_ssh_keys(self): return [self.ssh_public_key] diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 574243976..0d5c908b8 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -11,6 +11,7 @@ import copy import os import time +from typing import List from cloudinit import dmi from cloudinit import log as logging @@ -18,7 +19,8 @@ from cloudinit import url_helper as uhelp from cloudinit import util, warnings from cloudinit.event import EventScope, EventType -from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralIPNetwork from cloudinit.sources.helpers import ec2 LOG = logging.getLogger(__name__) @@ -67,7 +69,11 @@ class DataSourceEc2(sources.DataSource): # Priority ordered list of additional metadata versions which will be tried # for extended metadata content. IPv6 support comes in 2016-09-02. # Tags support comes in 2021-03-23. - extended_metadata_versions = ["2021-03-23", "2018-09-24", "2016-09-02"] + extended_metadata_versions: List[str] = [ + "2021-03-23", + "2018-09-24", + "2016-09-02", + ] # Setup read_url parameters per get_url_params. url_max_wait = 120 @@ -120,12 +126,16 @@ def _get_data(self): LOG.debug("FreeBSD doesn't support running dhclient with -sf") return False try: - with EphemeralDHCPv4(self.fallback_interface): + with EphemeralIPNetwork( + self.fallback_interface, ipv6=True + ) as netw: + state_msg = f" {netw.state_msg}" if netw.state_msg else "" self._crawled_metadata = util.log_time( logfunc=LOG.debug, - msg="Crawl of metadata service", + msg=f"Crawl of metadata service{state_msg}", func=self.crawl_metadata, ) + except NoDHCPLeaseError: return False else: @@ -222,7 +232,7 @@ def get_instance_id(self): else: return self.metadata["instance-id"] - def _maybe_fetch_api_token(self, mdurls, timeout=None, max_wait=None): + def _maybe_fetch_api_token(self, mdurls): """Get an API token for EC2 Instance Metadata Service. On EC2. IMDS will always answer an API token, unless @@ -472,12 +482,6 @@ def network_config(self): ), ) - # RELEASE_BLOCKER: xenial should drop the below if statement, - # because the issue being addressed doesn't exist pre-netplan. - # (This datasource doesn't implement check_instance_id() so the - # datasource object is recreated every boot; this means we don't - # need to modify update_events on cloud-init upgrade.) - # Non-VPC (aka Classic) Ec2 instances need to rewrite the # network config file every boot due to MAC address change. if self.is_classic_instance(): @@ -850,7 +854,7 @@ def convert_ec2_metadata_network_config( dev_config = { "dhcp4": True, "dhcp6": False, - "match": {"match": nicname}, + "match": {"match": nic_name}, "set-name": nic_name, } nic_metadata = macs_metadata.get(mac) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c470bea8f..3691a706e 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -11,7 +11,8 @@ from cloudinit import log as logging from cloudinit import sources, url_helper, util from cloudinit.distros import ug_util -from cloudinit.net.dhcp import EphemeralDHCPv4 +from cloudinit.net.ephemeral import EphemeralDHCPv4 +from cloudinit.sources import DataSourceHostname LOG = logging.getLogger(__name__) @@ -122,7 +123,9 @@ def publish_host_keys(self, hostkeys): def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): # GCE has long FDQN's and has asked for short hostnames. - return self.metadata["local-hostname"].split(".")[0] + return DataSourceHostname( + self.metadata["local-hostname"].split(".")[0], False + ) @property def availability_zone(self): diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 91a6f9c9b..90531769c 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -10,7 +10,8 @@ from cloudinit import dmi from cloudinit import log as logging from cloudinit import net, sources, util -from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralDHCPv4 LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py index 640348f49..34e4e00eb 100644 --- a/cloudinit/sources/DataSourceLXD.py +++ b/cloudinit/sources/DataSourceLXD.py @@ -13,22 +13,14 @@ import socket import stat from json.decoder import JSONDecodeError +from typing import Any, Dict, Union, cast import requests from requests.adapters import HTTPAdapter -# pylint fails to import the two modules below. -# These are imported via requests.packages rather than urllib3 because: -# a.) the provider of the requests package should ensure that urllib3 -# contained in it is consistent/correct. -# b.) cloud-init does not specifically have a dependency on urllib3 -# -# For future reference, see: -# https://github.com/kennethreitz/requests/pull/2375 -# https://github.com/requests/requests/issues/4104 -# pylint: disable=E0401 -from requests.packages.urllib3.connection import HTTPConnection -from requests.packages.urllib3.connectionpool import HTTPConnectionPool +# Note: `urllib3` is transitively installed by `requests`. +from urllib3.connection import HTTPConnection +from urllib3.connectionpool import HTTPConnectionPool from cloudinit import log as logging from cloudinit import sources, subp, util @@ -51,7 +43,7 @@ def generate_fallback_network_config() -> dict: """Return network config V1 dict representing instance network config.""" - network_v1 = { + network_v1: Dict[str, Any] = { "version": 1, "config": [ { @@ -142,8 +134,8 @@ class DataSourceLXD(sources.DataSource): dsname = "LXD" - _network_config = sources.UNSET - _crawled_metadata = sources.UNSET + _network_config: Union[Dict, str] = sources.UNSET + _crawled_metadata: Union[Dict, str] = sources.UNSET sensitive_metadata_keys = ( "merged_cfg", @@ -211,13 +203,17 @@ def network_config(self) -> dict: If none is present, then we generate fallback configuration. """ if self._network_config == sources.UNSET: - if self._crawled_metadata.get("network-config"): + if self._crawled_metadata == sources.UNSET: + self._get_data() + if isinstance( + self._crawled_metadata, dict + ) and self._crawled_metadata.get("network-config"): self._network_config = self._crawled_metadata.get( - "network-config" + "network-config", {} ) else: self._network_config = generate_fallback_network_config() - return self._network_config + return cast(dict, self._network_config) def is_platform_viable() -> bool: @@ -257,7 +253,7 @@ def read_metadata( configuration keys and values provided to the container surfaced by the socket under the /1.0/config/ route. """ - md = {} + md: dict = {} lxd_url = "http://lxd" version_url = lxd_url + "/" + api_version + "/" with requests.Session() as session: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index d5e2b323b..c5ad60a25 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -51,6 +51,10 @@ VMWARE_IMC_DIR = "/var/run/vmware-imc" +class GuestCustScriptDisabled(Exception): + pass + + class DataSourceOVF(sources.DataSource): dsname = "OVF" @@ -271,11 +275,20 @@ def _get_data(self): GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustErrorEnum.GUESTCUST_ERROR_SCRIPT_DISABLED, ) - raise RuntimeError(msg) + raise GuestCustScriptDisabled(msg) ccScriptsDir = os.path.join( self.paths.get_cpath("scripts"), "per-instance" ) + except GuestCustScriptDisabled as e: + LOG.debug("GuestCustScriptDisabled") + _raise_error_status( + "Error parsing the customization Config File", + e, + GuestCustErrorEnum.GUESTCUST_ERROR_SCRIPT_DISABLED, + vmwareImcConfigFilePath, + self._vmware_cust_conf, + ) except Exception as e: _raise_error_status( "Error parsing the customization Config File", diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 6878528df..292e0efc5 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -10,7 +10,8 @@ from cloudinit import log as logging from cloudinit import sources, url_helper, util from cloudinit.event import EventScope, EventType -from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralDHCPv4 from cloudinit.sources import DataSourceOracle as oracle from cloudinit.sources.helpers import openstack diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py index 3e21c0e0d..d489279ce 100644 --- a/cloudinit/sources/DataSourceOracle.py +++ b/cloudinit/sources/DataSourceOracle.py @@ -14,16 +14,17 @@ """ import base64 +import ipaddress from collections import namedtuple -from contextlib import suppress as noop -from typing import Tuple +from typing import Optional, Tuple from cloudinit import dmi from cloudinit import log as logging from cloudinit import net, sources, util +from cloudinit.distros.networking import NetworkConfig from cloudinit.net import ( cmdline, - dhcp, + ephemeral, get_interfaces_by_mac, is_netfail_master, ) @@ -46,7 +47,19 @@ OpcMetadata = namedtuple("OpcMetadata", "version instance_data vnics_data") -def _ensure_netfailover_safe(network_config): +class KlibcOracleNetworkConfigSource(cmdline.KlibcNetworkConfigSource): + """Override super class to lower the applicability conditions. + + If any `/run/net-*.cfg` files exist, then it is applicable. Even if + `/run/initramfs/open-iscsi.interface` does not exist. + """ + + def is_applicable(self) -> bool: + """Override is_applicable""" + return bool(self._files) + + +def _ensure_netfailover_safe(network_config: NetworkConfig) -> None: """ Search network config physical interfaces to see if any of them are a netfailover master. If found, we prevent matching by MAC as the other @@ -110,7 +123,7 @@ class DataSourceOracle(sources.DataSource): sources.NetworkConfigSource.SYSTEM_CFG, ) - _network_config = sources.UNSET + _network_config: dict = {"config": [], "version": 1} def __init__(self, sys_cfg, *args, **kwargs): super(DataSourceOracle, self).__init__(sys_cfg, *args, **kwargs) @@ -122,8 +135,12 @@ def __init__(self, sys_cfg, *args, **kwargs): BUILTIN_DS_CONFIG, ] ) + self._network_config_source = KlibcOracleNetworkConfigSource() + + def _has_network_config(self) -> bool: + return bool(self._network_config.get("config", [])) - def _is_platform_viable(self): + def _is_platform_viable(self) -> bool: """Check platform environment to report if this datasource may run.""" return _is_platform_viable() @@ -133,24 +150,21 @@ def _get_data(self): self.system_uuid = _read_system_uuid() - # network may be configured if iscsi root. If that is the case - # then read_initramfs_config will return non-None. - fetch_vnics_data = self.ds_cfg.get( + network_context = ephemeral.EphemeralDHCPv4( + iface=net.find_fallback_nic(), + connectivity_url_data={ + "url": METADATA_PATTERN.format(version=2, path="instance"), + "headers": V2_HEADERS, + }, + ) + fetch_primary_nic = not self._is_iscsi_root() + fetch_secondary_nics = self.ds_cfg.get( "configure_secondary_nics", BUILTIN_DS_CONFIG["configure_secondary_nics"], ) - network_context = noop() - if not _is_iscsi_root(): - network_context = dhcp.EphemeralDHCPv4( - iface=net.find_fallback_nic(), - connectivity_url_data={ - "url": METADATA_PATTERN.format(version=2, path="instance"), - "headers": V2_HEADERS, - }, - ) with network_context: fetched_metadata = read_opc_metadata( - fetch_vnics_data=fetch_vnics_data + fetch_vnics_data=fetch_primary_nic or fetch_secondary_nics ) data = self._crawled_metadata = fetched_metadata.instance_data @@ -177,7 +191,7 @@ def _get_data(self): return True - def check_instance_id(self, sys_cfg): + def check_instance_id(self, sys_cfg) -> bool: """quickly check (local only) if self.instance_id is still valid On Oracle, the dmi-provided system uuid differs from the instance-id @@ -187,59 +201,75 @@ def check_instance_id(self, sys_cfg): def get_public_ssh_keys(self): return sources.normalize_pubkey_data(self.metadata.get("public_keys")) + def _is_iscsi_root(self) -> bool: + """Return whether we are on a iscsi machine.""" + return self._network_config_source.is_applicable() + + def _get_iscsi_config(self) -> dict: + return self._network_config_source.render_config() + @property def network_config(self): """Network config is read from initramfs provided files + Priority for primary network_config selection: + - iscsi + - imds + If none is present, then we fall back to fallback configuration. """ - if self._network_config == sources.UNSET: - # this is v1 - self._network_config = cmdline.read_initramfs_config() - - if not self._network_config: - # this is now v2 - self._network_config = self.distro.generate_fallback_config() - - if self.ds_cfg.get( - "configure_secondary_nics", - BUILTIN_DS_CONFIG["configure_secondary_nics"], - ): - try: - # Mutate self._network_config to include secondary - # VNICs - self._add_network_config_from_opc_imds() - except Exception: - util.logexc( - LOG, "Failed to parse secondary network configuration!" - ) - - # we need to verify that the nic selected is not a netfail over - # device and, if it is a netfail master, then we need to avoid - # emitting any match by mac - _ensure_netfailover_safe(self._network_config) + if self._has_network_config(): + return self._network_config + + set_primary = False + # this is v1 + if self._is_iscsi_root(): + self._network_config = self._get_iscsi_config() + if not self._has_network_config(): + LOG.warning( + "Could not obtain network configuration from initramfs. " + "Falling back to IMDS." + ) + set_primary = True + + set_secondary = self.ds_cfg.get( + "configure_secondary_nics", + BUILTIN_DS_CONFIG["configure_secondary_nics"], + ) + if set_primary or set_secondary: + try: + # Mutate self._network_config to include primary and/or + # secondary VNICs + self._add_network_config_from_opc_imds(set_primary) + except Exception: + util.logexc( + LOG, + "Failed to parse IMDS network configuration!", + ) + + # we need to verify that the nic selected is not a netfail over + # device and, if it is a netfail master, then we need to avoid + # emitting any match by mac + _ensure_netfailover_safe(self._network_config) return self._network_config - def _add_network_config_from_opc_imds(self): - """Generate secondary NIC config from IMDS and merge it. + def _add_network_config_from_opc_imds(self, set_primary: bool = False): + """Generate primary and/or secondary NIC config from IMDS and merge it. - The primary NIC configuration should not be modified based on the IMDS - values, as it should continue to be configured for DHCP. As such, this - uses the instance's network config dict which is expected to have the - primary NIC configuration already present. It will mutate the network config to include the secondary VNICs. + :param set_primary: If True set primary interface. :raises: Exceptions are not handled within this function. Likely exceptions are KeyError/IndexError (if the IMDS returns valid JSON with unexpected contents). """ if self._vnics_data is None: - LOG.warning("Secondary NIC data is UNSET but should not be") + LOG.warning("NIC data is UNSET but should not be") return - if "nicIndex" in self._vnics_data[0]: + if not set_primary and ("nicIndex" in self._vnics_data[0]): # TODO: Once configure_secondary_nics defaults to True, lower the # level of this log message. (Currently, if we're running this # code at all, someone has explicitly opted-in to secondary @@ -255,57 +285,73 @@ def _add_network_config_from_opc_imds(self): interfaces_by_mac = get_interfaces_by_mac() - for vnic_dict in self._vnics_data[1:]: - # We skip the first entry in the response because the primary - # interface is already configured by iSCSI boot; applying - # configuration from the IMDS is not required. + vnics_data = self._vnics_data if set_primary else self._vnics_data[1:] + + for index, vnic_dict in enumerate(vnics_data): + is_primary = set_primary and index == 0 mac_address = vnic_dict["macAddr"].lower() if mac_address not in interfaces_by_mac: - LOG.debug( - "Interface with MAC %s not found; skipping", mac_address + LOG.warning( + "Interface with MAC %s not found; skipping", + mac_address, ) continue name = interfaces_by_mac[mac_address] + network = ipaddress.ip_network(vnic_dict["subnetCidrBlock"]) if self._network_config["version"] == 1: - subnet = { - "type": "static", - "address": vnic_dict["privateIp"], - } - self._network_config["config"].append( - { - "name": name, - "type": "physical", - "mac_address": mac_address, - "mtu": MTU, - "subnets": [subnet], + if is_primary: + subnet = {"type": "dhcp"} + else: + subnet = { + "type": "static", + "address": ( + f"{vnic_dict['privateIp']}/{network.prefixlen}" + ), } - ) + interface_config = { + "name": name, + "type": "physical", + "mac_address": mac_address, + "mtu": MTU, + "subnets": [subnet], + } + self._network_config["config"].append(interface_config) elif self._network_config["version"] == 2: - self._network_config["ethernets"][name] = { - "addresses": [vnic_dict["privateIp"]], + # Why does this elif exist??? + # Are there plans to switch to v2? + interface_config = { "mtu": MTU, - "dhcp4": False, - "dhcp6": False, "match": {"name": name}, + "dhcp6": False, + "dhcp4": is_primary, } + if not is_primary: + interface_config["addresses"] = [ + f"{vnic_dict['privateIp']}/{network.prefixlen}" + ] + self._network_config["ethernets"][name] = interface_config -def _read_system_uuid(): +def _read_system_uuid() -> Optional[str]: sys_uuid = dmi.read_dmi_data("system-uuid") return None if sys_uuid is None else sys_uuid.lower() -def _is_platform_viable(): +def _is_platform_viable() -> bool: asset_tag = dmi.read_dmi_data("chassis-asset-tag") return asset_tag == CHASSIS_ASSET_TAG -def _is_iscsi_root(): - return bool(cmdline.read_initramfs_config()) +def _fetch(metadata_version: int, path: str, retries: int = 2) -> dict: + return readurl( + url=METADATA_PATTERN.format(version=metadata_version, path=path), + headers=V2_HEADERS if metadata_version > 1 else None, + retries=retries, + )._response.json() -def read_opc_metadata(*, fetch_vnics_data: bool = False): +def read_opc_metadata(*, fetch_vnics_data: bool = False) -> OpcMetadata: """Fetch metadata from the /opc/ routes. :return: @@ -319,15 +365,6 @@ def read_opc_metadata(*, fetch_vnics_data: bool = False): # Per Oracle, there are short windows (measured in milliseconds) throughout # an instance's lifetime where the IMDS is being updated and may 404 as a # result. To work around these windows, we retry a couple of times. - retries = 2 - - def _fetch(metadata_version: int, path: str) -> dict: - return readurl( - url=METADATA_PATTERN.format(version=metadata_version, path=path), - headers=V2_HEADERS if metadata_version > 1 else None, - retries=retries, - )._response.json() - metadata_version = 2 try: instance_data = _fetch(metadata_version, path="instance") @@ -340,9 +377,7 @@ def _fetch(metadata_version: int, path: str) -> dict: try: vnics_data = _fetch(metadata_version, path="vnics") except UrlError: - util.logexc( - LOG, "Failed to fetch secondary network configuration!" - ) + util.logexc(LOG, "Failed to fetch IMDS network configuration!") return OpcMetadata(metadata_version, instance_data, vnics_data) diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index 14ac77e4d..6890562d7 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -12,6 +12,8 @@ import errno import os import os.path +import typing +from ipaddress import IPv4Address from cloudinit import log as logging from cloudinit import sources, subp, util @@ -30,18 +32,21 @@ def get_manage_etc_hosts(): return True -def ip2int(addr): - parts = addr.split(".") - return ( - (int(parts[0]) << 24) - + (int(parts[1]) << 16) - + (int(parts[2]) << 8) - + int(parts[3]) - ) +def increment_ip(addr, inc: int) -> str: + return str(IPv4Address(int(IPv4Address(addr)) + inc)) + +def get_three_ips(addr) -> typing.List[str]: + """Return a list of 3 IP addresses: [addr, addr + 2, addr + 3] -def int2ip(addr): - return ".".join([str(addr >> (i << 3) & 0xFF) for i in range(4)[::-1]]) + @param addr: an object that is passed to IPvAddress + @return: list of strings + """ + return [ + addr, + increment_ip(addr, 2), + increment_ip(addr, 3), + ] def _sub_arp(cmd): @@ -178,11 +183,7 @@ def read_user_data_callback(mount_dir): {"source": ip["address"], "destination": target} for netadp in meta_data["netadp"] for ip in netadp["ip"] - for target in [ - netadp["network"]["gateway"], - int2ip(ip2int(netadp["network"]["gateway"]) + 2), - int2ip(ip2int(netadp["network"]["gateway"]) + 3), - ] + for target in get_three_ips(netadp["network"]["gateway"]) ], "cfg": { "ssh_pwauth": True, diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py index c47a8bf55..0ba0dec3d 100644 --- a/cloudinit/sources/DataSourceScaleway.py +++ b/cloudinit/sources/DataSourceScaleway.py @@ -12,24 +12,17 @@ import requests -# pylint fails to import the two modules below. -# These are imported via requests.packages rather than urllib3 because: -# a.) the provider of the requests package should ensure that urllib3 -# contained in it is consistent/correct. -# b.) cloud-init does not specifically have a dependency on urllib3 -# -# For future reference, see: -# https://github.com/kennethreitz/requests/pull/2375 -# https://github.com/requests/requests/issues/4104 -# pylint: disable=E0401 -from requests.packages.urllib3.connection import HTTPConnection -from requests.packages.urllib3.poolmanager import PoolManager +# Note: `urllib3` is transitively installed by `requests` +from urllib3.connection import HTTPConnection +from urllib3.poolmanager import PoolManager from cloudinit import dmi from cloudinit import log as logging from cloudinit import net, sources, url_helper, util from cloudinit.event import EventScope, EventType -from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralDHCPv4 +from cloudinit.sources import DataSourceHostname LOG = logging.getLogger(__name__) @@ -288,7 +281,7 @@ def get_public_ssh_keys(self): return ssh_keys def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): - return self.metadata["hostname"] + return DataSourceHostname(self.metadata["hostname"], False) @property def availability_zone(self): diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 40f915fa6..11168f6a9 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -30,9 +30,11 @@ import re import socket +import serial + from cloudinit import dmi from cloudinit import log as logging -from cloudinit import serial, sources, subp, util +from cloudinit import sources, subp, util from cloudinit.event import EventScope, EventType LOG = logging.getLogger(__name__) @@ -711,8 +713,7 @@ def get(self, key, default=None, strip=False): if self.is_b64_encoded(key): try: val = base64.b64decode(val.encode()).decode() - # Bogus input produces different errors in Python 2 and 3 - except (TypeError, binascii.Error): + except binascii.Error: LOG.warning("Failed base64 decoding key '%s': %s", key, val) if strip: @@ -1049,7 +1050,7 @@ def load_key(client, key, data): return data[key] - data = {} + data: dict = {} for key in keys: load_key(client=jmc, key=key, data=data) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index f4b78da5e..d6b74bc14 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -8,7 +8,8 @@ from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import sources, util -from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralDHCPv4 from cloudinit.sources.helpers import upcloud as uc_helper LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py index 6ef7c9d51..308e02e8a 100644 --- a/cloudinit/sources/DataSourceVMware.py +++ b/cloudinit/sources/DataSourceVMware.py @@ -73,7 +73,7 @@ from cloudinit import dmi from cloudinit import log as logging -from cloudinit import sources, util +from cloudinit import net, sources, util from cloudinit.subp import ProcessExecutionError, subp, which PRODUCT_UUID_FILE_PATH = "/sys/class/dmi/id/product_uuid" @@ -685,20 +685,10 @@ def is_valid_ip_addr(val): Returns false if the address is loopback, link local or unspecified; otherwise true is returned. """ - # TODO(extend cloudinit.net.is_ip_addr exclude link_local/loopback etc) - # TODO(migrate to use cloudinit.net.is_ip_addr)# - - addr = None - try: - addr = ipaddress.ip_address(val) - except ipaddress.AddressValueError: - addr = ipaddress.ip_address(str(val)) - except Exception: - return None - - if addr.is_link_local or addr.is_loopback or addr.is_unspecified: - return False - return True + addr = net.maybe_get_address(ipaddress.ip_address, val) + return addr and not ( + addr.is_link_local or addr.is_loopback or addr.is_unspecified + ) def get_host_info(): @@ -810,7 +800,7 @@ def wait_on_network(metadata): wait_on_ipv6 = util.translate_bool(wait_on_ipv6_val) # Get information about the host. - host_info = None + host_info, ipv4_ready, ipv6_ready = None, False, False while host_info is None: # This loop + sleep results in two logs every second while waiting # for either ipv4 or ipv6 up. Do we really need to log each iteration @@ -855,7 +845,10 @@ def main(): except Exception: pass metadata = { - "wait-on-network": {"ipv4": True, "ipv6": "false"}, + WAIT_ON_NETWORK: { + WAIT_ON_NETWORK_IPV4: True, + WAIT_ON_NETWORK_IPV6: False, + }, "network": {"config": {"dhcp": True}}, } host_info = wait_on_network(metadata) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index fff760f1b..c399beb65 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -12,9 +12,10 @@ import copy import json import os +import pickle from collections import namedtuple from enum import Enum, unique -from typing import Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple from cloudinit import dmi, importer from cloudinit import log as logging @@ -149,7 +150,7 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): URLParams = namedtuple( - "URLParms", + "URLParams", [ "max_wait_seconds", "timeout_seconds", @@ -158,6 +159,11 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): ], ) +DataSourceHostname = namedtuple( + "DataSourceHostname", + ["hostname", "is_default"], +) + class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): @@ -228,7 +234,7 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): # N-tuple listing default values for any metadata-related class # attributes cached on an instance by a process_data runs. These attribute # values are reset via clear_cached_attrs during any update_metadata call. - cached_attr_defaults = ( + cached_attr_defaults: Tuple[Tuple[str, Any], ...] = ( ("ec2_metadata", UNSET), ("network_json", UNSET), ("metadata", {}), @@ -244,7 +250,7 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): # N-tuple of keypaths or keynames redact from instance-data.json for # non-root users - sensitive_metadata_keys = ( + sensitive_metadata_keys: Tuple[str, ...] = ( "merged_cfg", "security-credentials", ) @@ -256,7 +262,7 @@ def __init__(self, sys_cfg, distro: Distro, paths, ud_proc=None): self.distro = distro self.paths = paths self.userdata = None - self.metadata = {} + self.metadata: dict = {} self.userdata_raw = None self.vendordata = None self.vendordata2 = None @@ -301,7 +307,7 @@ def __str__(self): def _get_standardized_metadata(self, instance_data): """Return a dictionary of standardized metadata keys.""" - local_hostname = self.get_hostname() + local_hostname = self.get_hostname().hostname instance_id = self.get_instance_id() availability_zone = self.availability_zone # In the event of upgrade from existing cloudinit, pickled datasource @@ -356,7 +362,7 @@ def clear_cached_attrs(self, attr_defaults=()): if not attr_defaults: self._dirty_cache = False - def get_data(self): + def get_data(self) -> bool: """Datasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. @@ -368,14 +374,19 @@ def get_data(self): self.persist_instance_data() return return_value - def persist_instance_data(self): + def persist_instance_data(self, write_cache=True): """Process and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. + :param write_cache: boolean set True to persist obj.pkl when + instance_link exists. + @return True on successful write, False otherwise. """ + if write_cache and os.path.lexists(self.paths.instance_link): + pkl_store(self, self.paths.get_ipath_cur("obj_pkl")) if hasattr(self, "_crawled_metadata"): # Any datasource with _crawled_metadata will best represent # most recent, 'raw' metadata @@ -437,7 +448,7 @@ def persist_instance_data(self): write_json(json_file, redact_sensitive_keys(processed_data)) return True - def _get_data(self): + def _get_data(self) -> bool: """Walk metadata sources, process crawled data and save attributes.""" raise NotImplementedError( "Subclasses of DataSource must implement _get_data which" @@ -445,7 +456,7 @@ def _get_data(self): ) def get_url_params(self): - """Return the Datasource's prefered url_read parameters. + """Return the Datasource's preferred url_read parameters. Subclasses may override url_max_wait, url_timeout, url_retries. @@ -707,22 +718,33 @@ def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): @param metadata_only: Boolean, set True to avoid looking up hostname if meta-data doesn't have local-hostname present. - @return: hostname or qualified hostname. Optionally return None when + @return: a DataSourceHostname namedtuple + , (str, bool). + is_default is a bool and + it's true only if hostname is localhost and was + returned by util.get_hostname() as a default. + This is used to differentiate with a user-defined + localhost hostname. + Optionally return (None, False) when metadata_only is True and local-hostname data is not available. """ defdomain = "localdomain" defhost = "localhost" domain = defdomain + is_default = False if not self.metadata or not self.metadata.get("local-hostname"): if metadata_only: - return None + return DataSourceHostname(None, is_default) # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate # but instead, basically look up the existing hostname toks = [] hostname = util.get_hostname() + if hostname == "localhost": + # default hostname provided by socket.gethostname() + is_default = True hosts_fqdn = util.get_fqdn_from_hosts(hostname) if hosts_fqdn and hosts_fqdn.find(".") > 0: toks = str(hosts_fqdn).split(".") @@ -755,15 +777,15 @@ def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): hostname = toks[0] if fqdn and domain != defdomain: - return "%s.%s" % (hostname, domain) - else: - return hostname + hostname = "%s.%s" % (hostname, domain) + + return DataSourceHostname(hostname, is_default) def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) def get_supported_events(self, source_event_types: List[EventType]): - supported_events = {} # type: Dict[EventScope, set] + supported_events: Dict[EventScope, set] = {} for event in source_event_types: for ( update_scope, @@ -970,7 +992,9 @@ def list_sources(cfg_list, depends, pkg_list): return src_list -def instance_id_matches_system_uuid(instance_id, field="system-uuid"): +def instance_id_matches_system_uuid( + instance_id, field: str = "system-uuid" +) -> bool: # quickly (local check only) if self.instance_id is still valid # we check kernel command line or files. if not instance_id: @@ -1045,4 +1069,43 @@ def list_from_depends(depends, ds_list): return ret_list +def pkl_store(obj: DataSource, fname: str) -> bool: + """Use pickle to serialize Datasource to a file as a cache. + + :return: True on success + """ + try: + pk_contents = pickle.dumps(obj) + except Exception: + util.logexc(LOG, "Failed pickling datasource %s", obj) + return False + try: + util.write_file(fname, pk_contents, omode="wb", mode=0o400) + except Exception: + util.logexc(LOG, "Failed pickling datasource to %s", fname) + return False + return True + + +def pkl_load(fname: str) -> Optional[DataSource]: + """Use pickle to deserialize a instance Datasource from a cache file.""" + pickle_contents = None + try: + pickle_contents = util.load_file(fname, decode=False) + except Exception as e: + if os.path.isfile(fname): + LOG.warning("failed loading pickle in %s: %s", fname, e) + + # This is allowed so just return nothing successfully loaded... + if not pickle_contents: + return None + try: + return pickle.loads(pickle_contents) + except DatasourceUnpickleUserDataError: + return None + except Exception: + util.logexc(LOG, "Failed loading pickled blob from %s", fname) + return None + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 4bb8b8dbd..56f44339f 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 +import enum import json import logging import os @@ -7,16 +8,16 @@ import socket import struct import textwrap -import time import zlib from contextlib import contextmanager from datetime import datetime from errno import ENOENT -from typing import List, Optional +from time import sleep, time +from typing import List, Optional, Union from xml.etree import ElementTree from xml.sax.saxutils import escape -from cloudinit import distros, subp, temp_utils, url_helper, util, version +from cloudinit import distros, dmi, subp, temp_utils, url_helper, util, version from cloudinit.reporting import events from cloudinit.settings import CFG_BUILTIN @@ -99,7 +100,7 @@ def get_boot_telemetry(): LOG.debug("Collecting boot telemetry") try: - kernel_start = float(time.time()) - float(util.uptime()) + kernel_start = float(time()) - float(util.uptime()) except ValueError as e: raise RuntimeError("Failed to determine kernel start timestamp") from e @@ -331,45 +332,57 @@ def get_ip_from_lease_value(fallback_lease_value): @azure_ds_telemetry_reporter def http_with_retries( - url: str, *, headers: dict, data: Optional[str] = None + url: str, + *, + headers: dict, + data: Optional[str] = None, + retry_sleep: int = 5, + timeout_minutes: int = 20, ) -> url_helper.UrlResponse: """Readurl wrapper for querying wireserver. - Retries up to 40 minutes: - 240 attempts * (5s timeout + 5s sleep) + :param retry_sleep: Time to sleep before retrying. + :param timeout_minutes: Retry up to specified number of minutes. + :raises UrlError: on error fetching data. """ - max_readurl_attempts = 240 - readurl_timeout = 5 - sleep_duration_between_retries = 5 - periodic_logging_attempts = 12 + timeout = timeout_minutes * 60 + time() - for attempt in range(1, max_readurl_attempts + 1): + attempt = 0 + response = None + while not response: + attempt += 1 try: - ret = url_helper.readurl( - url, headers=headers, data=data, timeout=readurl_timeout + response = url_helper.readurl( + url, headers=headers, data=data, timeout=(5, 60) ) - + break + except url_helper.UrlError as e: report_diagnostic_event( - "Successful HTTP request with Azure endpoint %s after " - "%d attempts" % (url, attempt), + "Failed HTTP request with Azure endpoint %s during " + "attempt %d with exception: %s (code=%r headers=%r)" + % (url, attempt, e, e.code, e.headers), logger_func=LOG.debug, ) - - return ret - - except Exception as e: - if attempt % periodic_logging_attempts == 0: - report_diagnostic_event( - "Failed HTTP request with Azure endpoint %s during " - "attempt %d with exception: %s" % (url, attempt, e), - logger_func=LOG.debug, - ) - if attempt == max_readurl_attempts: + # Raise exception if we're out of time or network is unreachable. + # If network is unreachable: + # - retries will not resolve the situation + # - for reporting ready for PPS, this generally means VM was put + # to sleep or network interface was unplugged before we see + # the call complete successfully. + if ( + time() + retry_sleep >= timeout + or "Network is unreachable" in str(e) + ): raise - time.sleep(sleep_duration_between_retries) + sleep(retry_sleep) - raise RuntimeError("Failed to return in http_with_retries") + report_diagnostic_event( + "Successful HTTP request with Azure endpoint %s after " + "%d attempts" % (url, attempt), + logger_func=LOG.debug, + ) + return response def build_minimal_ovf( @@ -443,7 +456,7 @@ class InvalidGoalStateXMLException(Exception): class GoalState: def __init__( self, - unparsed_xml: str, + unparsed_xml: Union[str, bytes], azure_endpoint_client: AzureEndpointHttpClient, need_certificate: bool = True, ) -> None: @@ -777,11 +790,11 @@ def _post_health_report(self, document: str) -> None: # KVP messages that are published after the Azure Host receives the # signal are ignored and unprocessed, so yield this thread to the # Hyper-V KVP Reporting thread so that they are written. - # time.sleep(0) is a low-cost and proven method to yield the scheduler + # sleep(0) is a low-cost and proven method to yield the scheduler # and ensure that events are flushed. # See HyperVKvpReportingHandler class, which is a multi-threaded # reporting handler that writes to the special KVP files. - time.sleep(0) + sleep(0) LOG.debug("Sending health report to Azure fabric.") url = "http://{}/machine?comp=health".format(self._endpoint) @@ -883,7 +896,7 @@ def _fetch_goal_state_from_azure( ) @azure_ds_telemetry_reporter - def _get_raw_goal_state_xml_from_azure(self) -> str: + def _get_raw_goal_state_xml_from_azure(self) -> bytes: """Fetches the GoalState XML from the Azure endpoint and returns the XML as a string. @@ -911,7 +924,9 @@ def _get_raw_goal_state_xml_from_azure(self) -> str: @azure_ds_telemetry_reporter def _parse_raw_goal_state_xml( - self, unparsed_goal_state_xml: str, need_certificate: bool + self, + unparsed_goal_state_xml: Union[str, bytes], + need_certificate: bool, ) -> GoalState: """Parses a GoalState XML string and returns a GoalState object. @@ -1055,4 +1070,239 @@ def dhcp_log_cb(out, err): ) +class BrokenAzureDataSource(Exception): + pass + + +class NonAzureDataSource(Exception): + pass + + +class ChassisAssetTag(enum.Enum): + AZURE_CLOUD = "7783-7084-3265-9085-8269-3286-77" + + @classmethod + def query_system(cls) -> Optional["ChassisAssetTag"]: + """Check platform environment to report if this datasource may run. + + :returns: ChassisAssetTag if matching tag found, else None. + """ + asset_tag = dmi.read_dmi_data("chassis-asset-tag") + try: + tag = cls(asset_tag) + except ValueError: + report_diagnostic_event( + "Non-Azure chassis asset tag: %r" % asset_tag, + logger_func=LOG.debug, + ) + return None + + report_diagnostic_event( + "Azure chassis asset tag: %r (%s)" % (asset_tag, tag.name), + logger_func=LOG.debug, + ) + return tag + + +class OvfEnvXml: + NAMESPACES = { + "ovf": "http://schemas.dmtf.org/ovf/environment/1", + "wa": "http://schemas.microsoft.com/windowsazure", + } + + def __init__( + self, + *, + username: Optional[str] = None, + password: Optional[str] = None, + hostname: Optional[str] = None, + custom_data: Optional[bytes] = None, + disable_ssh_password_auth: Optional[bool] = None, + public_keys: Optional[List[dict]] = None, + preprovisioned_vm: bool = False, + preprovisioned_vm_type: Optional[str] = None, + ) -> None: + self.username = username + self.password = password + self.hostname = hostname + self.custom_data = custom_data + self.disable_ssh_password_auth = disable_ssh_password_auth + self.public_keys: List[dict] = public_keys or [] + self.preprovisioned_vm = preprovisioned_vm + self.preprovisioned_vm_type = preprovisioned_vm_type + + def __eq__(self, other) -> bool: + return self.__dict__ == other.__dict__ + + @classmethod + def parse_text(cls, ovf_env_xml: str) -> "OvfEnvXml": + """Parser for ovf-env.xml data. + + :raises NonAzureDataSource: if XML is not in Azure's format. + :raises BrokenAzureDataSource: if XML is unparseable or invalid. + """ + try: + root = ElementTree.fromstring(ovf_env_xml) + except ElementTree.ParseError as e: + error_str = "Invalid ovf-env.xml: %s" % e + raise BrokenAzureDataSource(error_str) from e + + # If there's no provisioning section, it's not Azure ovf-env.xml. + if not root.find("./wa:ProvisioningSection", cls.NAMESPACES): + raise NonAzureDataSource( + "Ignoring non-Azure ovf-env.xml: ProvisioningSection not found" + ) + + instance = OvfEnvXml() + instance._parse_linux_configuration_set_section(root) + instance._parse_platform_settings_section(root) + + return instance + + def _find( + self, + node, + name: str, + required: bool, + namespace: str = "wa", + ): + matches = node.findall( + "./%s:%s" % (namespace, name), OvfEnvXml.NAMESPACES + ) + if len(matches) == 0: + msg = "No ovf-env.xml configuration for %r" % name + LOG.debug(msg) + if required: + raise BrokenAzureDataSource(msg) + return None + elif len(matches) > 1: + raise BrokenAzureDataSource( + "Multiple configuration matches in ovf-exml.xml for %r (%d)" + % (name, len(matches)) + ) + + return matches[0] + + def _parse_property( + self, + node, + name: str, + required: bool, + decode_base64: bool = False, + parse_bool: bool = False, + default=None, + ): + matches = node.findall("./wa:" + name, OvfEnvXml.NAMESPACES) + if len(matches) == 0: + msg = "No ovf-env.xml configuration for %r" % name + LOG.debug(msg) + if required: + raise BrokenAzureDataSource(msg) + return default + elif len(matches) > 1: + raise BrokenAzureDataSource( + "Multiple configuration matches in ovf-exml.xml for %r (%d)" + % (name, len(matches)) + ) + + value = matches[0].text + + # Empty string may be None. + if value is None: + value = default + + if decode_base64 and value is not None: + value = base64.b64decode("".join(value.split())) + + if parse_bool: + value = util.translate_bool(value) + + return value + + def _parse_linux_configuration_set_section(self, root): + provisioning_section = self._find( + root, "ProvisioningSection", required=True + ) + config_set = self._find( + provisioning_section, + "LinuxProvisioningConfigurationSet", + required=True, + ) + + self.custom_data = self._parse_property( + config_set, + "CustomData", + decode_base64=True, + required=False, + ) + self.username = self._parse_property( + config_set, "UserName", required=True + ) + self.password = self._parse_property( + config_set, "UserPassword", required=False + ) + self.hostname = self._parse_property( + config_set, "HostName", required=True + ) + self.disable_ssh_password_auth = self._parse_property( + config_set, + "DisableSshPasswordAuthentication", + parse_bool=True, + required=False, + ) + + self._parse_ssh_section(config_set) + + def _parse_platform_settings_section(self, root): + platform_settings_section = self._find( + root, "PlatformSettingsSection", required=True + ) + platform_settings = self._find( + platform_settings_section, "PlatformSettings", required=True + ) + + self.preprovisioned_vm = self._parse_property( + platform_settings, + "PreprovisionedVm", + parse_bool=True, + default=False, + required=False, + ) + self.preprovisioned_vm_type = self._parse_property( + platform_settings, + "PreprovisionedVMType", + required=False, + ) + + def _parse_ssh_section(self, config_set): + self.public_keys = [] + + ssh_section = self._find(config_set, "SSH", required=False) + if ssh_section is None: + return + + public_keys_section = self._find( + ssh_section, "PublicKeys", required=False + ) + if public_keys_section is None: + return + + for public_key in public_keys_section.findall( + "./wa:PublicKey", OvfEnvXml.NAMESPACES + ): + fingerprint = self._parse_property( + public_key, "Fingerprint", required=False + ) + path = self._parse_property(public_key, "Path", required=False) + value = self._parse_property( + public_key, "Value", default="", required=False + ) + ssh_key = { + "fingerprint": fingerprint, + "path": path, + "value": value, + } + self.public_keys.append(ssh_key) + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/cloudsigma.py b/cloudinit/sources/helpers/cloudsigma.py index 6db7e117b..19fa1669a 100644 --- a/cloudinit/sources/helpers/cloudsigma.py +++ b/cloudinit/sources/helpers/cloudsigma.py @@ -22,7 +22,7 @@ import json import platform -from cloudinit import serial +import serial # these high timeouts are necessary as read may read a lot of data. READ_TIMEOUT = 60 diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 845294ec3..4def10f10 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -5,13 +5,9 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import configparser import logging -try: - import configparser -except ImportError: - import ConfigParser as configparser - from .config_source import ConfigSource logger = logging.getLogger(__name__) diff --git a/cloudinit/sources/helpers/vultr.py b/cloudinit/sources/helpers/vultr.py index c8fb8420b..adbcfbe58 100644 --- a/cloudinit/sources/helpers/vultr.py +++ b/cloudinit/sources/helpers/vultr.py @@ -10,7 +10,8 @@ from cloudinit import dmi from cloudinit import log as log from cloudinit import net, netinfo, subp, url_helper, util -from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralDHCPv4 # Get LOG LOG = log.getLogger(__name__) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 2a73ca82a..6d186992e 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -403,9 +403,11 @@ def check_create_path(username, filename, strictmodes): return True + def not_cli(path): return "/cli/" not in path + def extract_authorized_keys(username, sshd_cfg_file=DEF_SSHD_CFG): (ssh_dir, pw_ent) = users_ssh_info(username) default_authorizedkeys_file = os.path.join(ssh_dir, "authorized_keys") @@ -549,11 +551,28 @@ def parse_ssh_config_map(fname): return ret +def _includes_dconf(fname: str) -> bool: + if not os.path.isfile(fname): + return False + with open(fname, "r") as f: + for line in f: + if line.startswith(f"Include {fname}.d/*.conf"): + return True + return False + + def update_ssh_config(updates, fname=DEF_SSHD_CFG): """Read fname, and update if changes are necessary. @param updates: dictionary of desired values {Option: value} @return: boolean indicating if an update was done.""" + if _includes_dconf(fname): + if not os.path.isdir(f"{fname}.d"): + util.ensure_dir(f"{fname}.d", mode=0o755) + fname = os.path.join(f"{fname}.d", "50-cloud-init.conf") + if not os.path.isfile(fname): + # Ensure root read-only: + util.ensure_file(fname, 0o600) lines = parse_ssh_config(fname) changed = update_ssh_config_lines(lines=lines, updates=updates) if changed: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 27af60551..132dd83ba 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -6,7 +6,6 @@ import copy import os -import pickle import sys from collections import namedtuple from typing import Dict, Iterable, List, Optional, Set @@ -247,7 +246,7 @@ def _restore_from_cache(self): # We try to restore from a current link and static path # by using the instance link, if purge_cache was called # the file wont exist. - return _pkl_load(self.paths.get_ipath_cur("obj_pkl")) + return sources.pkl_load(self.paths.get_ipath_cur("obj_pkl")) def _write_to_cache(self): if self.datasource is None: @@ -260,7 +259,9 @@ def _write_to_cache(self): omode="w", content="", ) - return _pkl_store(self.datasource, self.paths.get_ipath_cur("obj_pkl")) + return sources.pkl_store( + self.datasource, self.paths.get_ipath_cur("obj_pkl") + ) def _get_datasources(self): # Any config provided??? @@ -576,7 +577,7 @@ def register_handlers_in_dir(path): # Attempts to register any handler modules under the given path. if not path or not os.path.isdir(path): return - potential_handlers = util.find_modules(path) + potential_handlers = util.get_modules_from_dir(path) for (fname, mod_name) in potential_handlers.items(): try: mod_locs, looked_locs = importer.find_module( @@ -973,38 +974,4 @@ def fetch_base_config(): ) -def _pkl_store(obj, fname): - try: - pk_contents = pickle.dumps(obj) - except Exception: - util.logexc(LOG, "Failed pickling datasource %s", obj) - return False - try: - util.write_file(fname, pk_contents, omode="wb", mode=0o400) - except Exception: - util.logexc(LOG, "Failed pickling datasource to %s", fname) - return False - return True - - -def _pkl_load(fname): - pickle_contents = None - try: - pickle_contents = util.load_file(fname, decode=False) - except Exception as e: - if os.path.isfile(fname): - LOG.warning("failed loading pickle in %s: %s", fname, e) - - # This is allowed so just return nothing successfully loaded... - if not pickle_contents: - return None - try: - return pickle.loads(pickle_contents) - except sources.DatasourceUnpickleUserDataError: - return None - except Exception: - util.logexc(LOG, "Failed loading pickled blob from %s", fname) - return None - - # vi: ts=4 expandtab diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 298eaf6b9..4d712829e 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -10,31 +10,38 @@ # # This file is part of cloud-init. See LICENSE file for license information. +# noqa: E402 + import collections import re import sys +from typing import Type + +from cloudinit import log as logging +from cloudinit import type_utils as tu +from cloudinit import util +from cloudinit.atomic_helper import write_file +JUndefined: Type try: - from jinja2 import DebugUndefined as JUndefined + from jinja2 import DebugUndefined as _DebugUndefined from jinja2 import Template as JTemplate JINJA_AVAILABLE = True + JUndefined = _DebugUndefined except (ImportError, AttributeError): JINJA_AVAILABLE = False JUndefined = object -from cloudinit import log as logging -from cloudinit import type_utils as tu -from cloudinit import util -from cloudinit.atomic_helper import write_file - LOG = logging.getLogger(__name__) TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) BASIC_MATCHER = re.compile(r"\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)") MISSING_JINJA_PREFIX = "CI_MISSING_JINJA_VAR/" -class UndefinedJinjaVariable(JUndefined): +# Mypy, and the PEP 484 ecosystem in general, does not support creating +# classes with dynamic base types: https://stackoverflow.com/a/59636248 +class UndefinedJinjaVariable(JUndefined): # type: ignore """Class used to represent any undefined jinja template variable.""" def __str__(self): diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 04643895e..291b8d4df 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -19,7 +19,7 @@ from functools import partial from http.client import NOT_FOUND from itertools import count -from typing import Any, Callable, List, Tuple +from typing import Any, Callable, Iterator, List, Optional, Tuple, Union from urllib.parse import quote, urlparse, urlunparse import requests @@ -59,7 +59,7 @@ def combine_single(url, add_on): return url -def read_file_or_url(url, **kwargs): +def read_file_or_url(url, **kwargs) -> Union["FileResponse", "UrlResponse"]: """Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. @@ -113,11 +113,13 @@ def __init__(self, path, contents, code=200): class UrlResponse(object): - def __init__(self, response): + def __init__(self, response: requests.Response): self._response = response @property - def contents(self): + def contents(self) -> bytes: + if self._response.content is None: + return b"" return self._response.content @property @@ -144,6 +146,20 @@ def code(self): def __str__(self): return self._response.text + def iter_content( + self, chunk_size: Optional[int] = 1, decode_unicode: bool = False + ) -> Iterator[bytes]: + """Iterates over the response data. + + When stream=True is set on the request, this avoids reading the content + at once into memory for large responses. + + :param chunk_size: Number of bytes it should read into memory. + :param decode_unicode: If True, content will be decoded using the best + available encoding based on the response. + """ + yield from self._response.iter_content(chunk_size, decode_unicode) + class UrlError(IOError): def __init__(self, cause, code=None, headers=None, url=None): @@ -191,6 +207,7 @@ def readurl( infinite=False, log_req_resp=True, request_method="", + stream: bool = False, ) -> UrlResponse: """Wrapper around requests.Session to read the url and retry if necessary @@ -222,10 +239,13 @@ def readurl( :param request_method: String passed as 'method' to Session.request. Typically GET, or POST. Default: POST if data is provided, GET otherwise. + :param stream: if False, the response content will be immediately + downloaded. """ url = _cleanurl(url) req_args = { "url": url, + "stream": stream, } req_args.update(_get_ssl_args(url, ssl_details)) req_args["allow_redirects"] = allow_redirects @@ -443,7 +463,7 @@ def dual_stack( "Timed out waiting for addresses: %s, " "exception(s) raised while waiting: %s", " ".join(addresses), - " ".join(exceptions), + " ".join(exceptions), # type: ignore ) finally: executor.shutdown(wait=False) @@ -460,7 +480,7 @@ def wait_for_url( headers_redact=None, sleep_time: int = 1, exception_cb: Callable = None, - sleep_time_cb: Callable = None, + sleep_time_cb: Callable[[Any, int], int] = None, request_method: str = "", connect_synchronously: bool = True, async_delay: float = 0.150, @@ -503,7 +523,7 @@ def wait_for_url( A value of None for max_wait will retry indefinitely. """ - def default_sleep_time(_, loop_number: int): + def default_sleep_time(_, loop_number: int) -> int: return int(loop_number / 5) + 1 def timeup(max_wait, start_time): @@ -631,9 +651,7 @@ def read_url_parallel(start_time, timeout, exc_cb, log_cb): read_url_serial if connect_synchronously else read_url_parallel ) - calculate_sleep_time = ( - default_sleep_time if not sleep_time_cb else sleep_time_cb - ) + calculate_sleep_time = sleep_time_cb or default_sleep_time loop_n: int = 0 response = None diff --git a/cloudinit/util.py b/cloudinit/util.py index 2639478af..77e7f66b1 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -32,10 +32,10 @@ import sys import time from base64 import b64decode, b64encode -from collections import deque +from collections import deque, namedtuple from errno import EACCES, ENOENT -from functools import lru_cache -from typing import List +from functools import lru_cache, total_ordering +from typing import Callable, List, TypeVar from urllib import parse from cloudinit import importer @@ -368,7 +368,7 @@ def extract_usergroup(ug_pair): return (u, g) -def find_modules(root_dir) -> dict: +def get_modules_from_dir(root_dir: str) -> dict: entries = dict() for fname in glob.glob(os.path.join(root_dir, "*.py")): if not os.path.isfile(fname): @@ -601,6 +601,7 @@ def _get_variant(info): "fedora", "miraclelinux", "openeuler", + "openmandriva", "photon", "rhel", "rocky", @@ -800,28 +801,6 @@ def set_subprocess_umask_and_gid(): os.dup2(new_fp.fileno(), o_err.fileno()) -def make_url( - scheme, host, port=None, path="", params="", query="", fragment="" -): - - pieces = [scheme or ""] - - netloc = "" - if host: - netloc = str(host) - - if port is not None: - netloc += ":" + "%s" % (port) - - pieces.append(netloc or "") - pieces.append(path or "") - pieces.append(params or "") - pieces.append(query or "") - pieces.append(fragment or "") - - return parse.urlunparse(pieces) - - def mergemanydict(srcs, reverse=False) -> dict: if reverse: srcs = reversed(srcs) @@ -887,17 +866,16 @@ def read_optional_seed(fill, base="", ext="", timeout=5): def fetch_ssl_details(paths=None): ssl_details = {} # Lookup in these locations for ssl key/cert files - ssl_cert_paths = [ - "/var/lib/cloud/data/ssl", - "/var/lib/cloud/instance/data/ssl", - ] - if paths: - ssl_cert_paths.extend( - [ - os.path.join(paths.get_ipath_cur("data"), "ssl"), - os.path.join(paths.get_cpath("data"), "ssl"), - ] - ) + if not paths: + ssl_cert_paths = [ + "/var/lib/cloud/data/ssl", + "/var/lib/cloud/instance/data/ssl", + ] + else: + ssl_cert_paths = [ + os.path.join(paths.get_ipath_cur("data"), "ssl"), + os.path.join(paths.get_cpath("data"), "ssl"), + ] ssl_cert_paths = uniq_merge(ssl_cert_paths) ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] cert_file = None @@ -1103,6 +1081,12 @@ def dos2unix(contents): return contents.replace("\r\n", "\n") +HostnameFqdnInfo = namedtuple( + "HostnameFqdnInfo", + ["hostname", "fqdn", "is_default"], +) + + def get_hostname_fqdn(cfg, cloud, metadata_only=False): """Get hostname and fqdn from config if present and fallback to cloud. @@ -1110,9 +1094,17 @@ def get_hostname_fqdn(cfg, cloud, metadata_only=False): @param cloud: Cloud instance from init.cloudify(). @param metadata_only: Boolean, set True to only query cloud meta-data, returning None if not present in meta-data. - @return: a Tuple of strings , . Values can be none when + @return: a namedtuple of + , , (str, str, bool). + Values can be none when metadata_only is True and no cfg or metadata provides hostname info. + is_default is a bool and + it's true only if hostname is localhost and was + returned by util.get_hostname() as a default. + This is used to differentiate with a user-defined + localhost hostname. """ + is_default = False if "fqdn" in cfg: # user specified a fqdn. Default hostname then is based off that fqdn = cfg["fqdn"] @@ -1126,12 +1118,16 @@ def get_hostname_fqdn(cfg, cloud, metadata_only=False): else: # no fqdn set, get fqdn from cloud. # get hostname from cfg if available otherwise cloud - fqdn = cloud.get_hostname(fqdn=True, metadata_only=metadata_only) + fqdn = cloud.get_hostname( + fqdn=True, metadata_only=metadata_only + ).hostname if "hostname" in cfg: hostname = cfg["hostname"] else: - hostname = cloud.get_hostname(metadata_only=metadata_only) - return (hostname, fqdn) + hostname, is_default = cloud.get_hostname( + metadata_only=metadata_only + ) + return HostnameFqdnInfo(hostname, fqdn, is_default) def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): @@ -1723,37 +1719,15 @@ def json_serialize_default(_obj): return "Warning: redacted unserializable type {0}".format(type(_obj)) -def json_preserialize_binary(data): - """Preserialize any discovered binary values to avoid json.dumps issues. - - Used only on python 2.7 where default type handling is not honored for - failure to encode binary data. LP: #1801364. - TODO(Drop this function when py2.7 support is dropped from cloud-init) - """ - data = obj_copy.deepcopy(data) - for key, value in data.items(): - if isinstance(value, (dict)): - data[key] = json_preserialize_binary(value) - if isinstance(value, bytes): - data[key] = "ci-b64:{0}".format(b64e(value)) - return data - - def json_dumps(data): """Return data in nicely formatted json.""" - try: - return json.dumps( - data, - indent=1, - sort_keys=True, - separators=(",", ": "), - default=json_serialize_default, - ) - except UnicodeDecodeError: - if sys.version_info[:2] == (2, 7): - data = json_preserialize_binary(data) - return json.dumps(data) - raise + return json.dumps( + data, + indent=1, + sort_keys=True, + separators=(",", ": "), + default=json_serialize_default, + ) def ensure_dir(path, mode=None): @@ -2618,7 +2592,17 @@ def get_mount_info(path, log=LOG, get_mnt_opts=False): return parse_mount(path) -def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): +T = TypeVar("T") + + +def log_time( + logfunc, + msg, + func: Callable[..., T], + args=None, + kwargs=None, + get_uptime=False, +) -> T: if args is None: args = [] if kwargs is None: @@ -2800,14 +2784,6 @@ def system_is_snappy(): return False -def indent(text, prefix): - """replacement for indent from textwrap that is not available in 2.7.""" - lines = [] - for line in text.splitlines(True): - lines.append(prefix + line) - return "".join(lines) - - def rootdev_from_cmdline(cmdline): found = None for tok in cmdline.split(): @@ -2918,13 +2894,19 @@ def get_proc_ppid(pid): ppid = 0 try: contents = load_file("/proc/%s/stat" % pid, quiet=True) + if contents: + # see proc.5 for format + m = re.search(r"^\d+ \(.+\) [RSDZTtWXxKPI] (\d+)", str(contents)) + if m: + ppid = int(m.group(1)) + else: + LOG.warning( + "Unable to match parent pid of process pid=%s input: %s", + pid, + contents, + ) except IOError as e: LOG.warning("Failed to load /proc/%s/stat. %s", pid, e) - if contents: - parts = contents.split(" ", 4) - # man proc says - # ppid %d (4) The PID of the parent. - ppid = int(parts[3]) return ppid @@ -2943,4 +2925,46 @@ def error(msg, rc=1, fmt="Error:\n{}", sys_exit=False): return rc -# vi: ts=4 expandtab +@total_ordering +class Version(namedtuple("Version", ["major", "minor", "patch", "rev"])): + def __new__(cls, major=-1, minor=-1, patch=-1, rev=-1): + """Default of -1 allows us to tiebreak in favor of the most specific + number""" + return super(Version, cls).__new__(cls, major, minor, patch, rev) + + @classmethod + def from_str(cls, version: str): + return cls(*(list(map(int, version.split("."))))) + + def __gt__(self, other): + return 1 == self._compare_version(other) + + def __eq__(self, other): + return ( + self.major == other.major + and self.minor == other.minor + and self.patch == other.patch + and self.rev == other.rev + ) + + def _compare_version(self, other) -> int: + """ + return values: + 1: self > v2 + -1: self < v2 + 0: self == v2 + + to break a tie between 3.1.N and 3.1, always treat the more + specific number as larger + """ + if self == other: + return 0 + if self.major > other.major: + return 1 + if self.minor > other.minor: + return 1 + if self.patch > other.patch: + return 1 + if self.rev > other.rev: + return 1 + return -1 diff --git a/cloudinit/version.py b/cloudinit/version.py index 061ea419e..711bf1710 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "22.2" +__VERSION__ = "22.3.4" _PACKAGED_VERSION = "@@PACKAGED_VERSION@@" FEATURES = [ diff --git a/config/clean.d/README b/config/clean.d/README new file mode 100644 index 000000000..9b0feebea --- /dev/null +++ b/config/clean.d/README @@ -0,0 +1,18 @@ +-- cloud-init's clean.d run-parts directory -- + +This directory is provided for third party applications which need +additional configuration artifact cleanup from the filesystem when +the command `cloud-init clean` is invoked. + +The `cloud-init clean` operation is typically performed by image creators +when preparing a golden image for clone and redeployment. The clean command +removes any cloud-init semaphores, allowing cloud-init to treat the next +boot of this image as the "first boot". When the image is next booted +cloud-init will performing all initial configuration based on any valid +datasource meta-data and user-data. + +Any executable scripts in this subdirectory will be invoked in lexicographical +order with run-parts by the command: sudo cloud-init clean. + +Typical format of such scripts would be a ##- like the following: + /etc/cloud/clean.d/99-live-installer diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 6d6e7ed71..d07f9faa3 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -33,7 +33,7 @@ disable_root: true {% endif %} {% if variant in ["almalinux", "alpine", "amazon", "centos", "cloudlinux", "eurolinux", - "fedora", "miraclelinux", "openEuler", "rhel", "rocky", "virtuozzo"] %} + "fedora", "miraclelinux", "openEuler", "openmandriva", "rhel", "rocky", "virtuozzo"] %} {% if variant == "rhel" %} mount_default_fields: [~, ~, 'auto', 'defaults,nofail,x-systemd.requires=cloud-init.service,_netdev', '0', '2'] {% else %} @@ -43,7 +43,7 @@ mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] resize_rootfs: noblock {% endif %} resize_rootfs_tmp: /dev -ssh_pwauth: 0 +ssh_pwauth: false {% endif %} # This will cause the set+update hostname module to not operate (if true) @@ -110,9 +110,15 @@ cloud_init_modules: # The modules that run in the 'config' stage cloud_config_modules: +{% if variant in ["ubuntu"] %} + - wireguard +{% endif %} {% if variant in ["ubuntu", "unknown", "debian"] %} - snap {% endif %} +{% if variant in ["ubuntu"] %} + - ubuntu_autoinstall +{% endif %} {% if variant not in ["photon"] %} - ssh-import-id {% if variant not in ["rhel"] %} @@ -124,7 +130,7 @@ cloud_config_modules: {% if variant in ["rhel"] %} - rh_subscription {% endif %} -{% if variant in ["rhel", "fedora", "photon"] %} +{% if variant in ["rhel", "fedora", "openmandriva", "photon"] %} {% if variant not in ["photon"] %} - spacewalk {% endif %} @@ -167,6 +173,7 @@ cloud_final_modules: - write-files-deferred - puppet - chef + - ansible - mcollective - salt-minion - reset_rmc @@ -189,7 +196,7 @@ system_info: # This will affect which distro class gets used {% if variant in ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "debian", "eurolinux", "fedora", "freebsd", "gentoo", "netbsd", "miraclelinux", "openbsd", "openEuler", - "photon", "rhel", "rocky", "suse", "ubuntu", "virtuozzo"] %} + "openmandriva", "photon", "rhel", "rocky", "suse", "ubuntu", "virtuozzo"] %} distro: {{ variant }} {% elif variant in ["dragonfly"] %} distro: dragonflybsd @@ -209,6 +216,7 @@ system_info: {# SRU_BLOCKER: do not ship network renderers on Xenial, Bionic or Eoan #} network: renderers: ['netplan', 'eni', 'sysconfig'] + activators: ['netplan', 'eni', 'network-manager', 'networkd'] # Automatically discover the best ntp_client ntp_client: auto # Other config here will be given to the distro class and/or path classes @@ -242,7 +250,7 @@ system_info: security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh {% elif variant in ["almalinux", "alpine", "amazon", "arch", "centos", "cloudlinux", "eurolinux", - "fedora", "gentoo", "miraclelinux", "openEuler", "rhel", "rocky", "suse", "virtuozzo"] %} + "fedora", "gentoo", "miraclelinux", "openEuler", "openmandriva", "rhel", "rocky", "suse", "virtuozzo"] %} # Default user name + that default users groups (if added/used) default_user: {% if variant == "amazon" %} @@ -253,6 +261,10 @@ system_info: name: cloud-user lock_passwd: true gecos: Cloud User +{% elif variant == "openmandriva" %} + name: omv + lock_passwd: True + gecos: OpenMandriva admin {% else %} name: {{ variant }} lock_passwd: True @@ -268,6 +280,8 @@ system_info: groups: [adm, sudo] {% elif variant == "arch" %} groups: [wheel, users] +{% elif variant == "openmandriva" %} + groups: [wheel, users, systemd-journal] {% elif variant == "rhel" %} groups: [adm, systemd-journal] {% else %} @@ -346,6 +360,12 @@ system_info: {% elif variant in ["dragonfly"] %} network: renderers: ['freebsd'] +{% elif variant in ["rhel", "fedora"] %} + network: + renderers: ['netplan', 'network-manager', 'networkd', 'sysconfig', 'eni'] +{% elif variant == "openmandriva" %} + network: + renderers: ['network-manager', 'networkd'] {% endif %} disable_vmware_customization: false diff --git a/debian/changelog b/debian/changelog index 96ad32540..2bc2292e0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,294 @@ -cloud-init (22.2-0ubuntu1~20.04.3) focal-security; urgency=medium +cloud-init (22.3.4-0ubuntu1~20.04.1) focal; urgency=medium - * SECURITY UPDATE: schema errors can cause cloud-init to leak - userdata to system logs - - d/cloud-init.postinst: redact previously leaked schema errors - from logs - - Remove schema errors from log (LP: #1978422) - - CVE-2022-2084 + * New upstream bugfix release. (LP: #1987318) + + Release 22.3.4 (LP: #1986703) + + Fix Oracle DS primary interface when using IMDS (#1757) + (LP: #1989686) + + -- Brett Holman Mon, 03 Oct 2022 10:57:17 -0600 + +cloud-init (22.3.3-0ubuntu1~20.04.1) focal; urgency=medium + + * New upstream bugfix release. (LP: #1987318) + + Release 22.3.3 + + Fix Oracle DS not setting subnet when using IMDS (#1735) + + azure: define new attribute for pre-22.3 pickles (#1725) + + sources/azure: ensure instance id is always correct (#1727) + [Chris Patterson] + + -- Brett Holman Wed, 21 Sep 2022 14:16:25 -0600 + +cloud-init (22.3-13-g70ce6442-0ubuntu1~20.04.1) focal; urgency=medium + + * d/control: + - add python3-debconf to Depends and Build-Depends + - Build-Depends: bump debhelper-compat to v10 + * d/control: lintian fixes: + + upgrade debhelper-compat to 10 and move it to d/control + + d/compat: removed in favor of d/control + * d/cloud-init.postinst: + + Lintian: Disable uses-dpkg-database-directly on legit use of it in + distros/debian.py + * d/cloud-init.postinst: lintian fixes: + + Fix command-with-path-in-maintainer-script for grub-install + * d/p/expire-on-hashed-users.patch: + Add patch to ensure password expire doesn't apply to hashed users + * d/source/lintian-overrides: lintian fixes: + + silence binary-nmu-debian-revision-in-source bug: + https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1014584 + * drop the following cherry-picks now included: + + cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500 + * New upstream snapshot. (LP: #1987318) + + Fix v2 interface matching when no MAC + + test: reduce number of network dependencies in flaky test (#1702) + + docs: publish cc_ubuntu_autoinstall docs to rtd (#1696) + + net: Fix EphemeraIPNetwork (#1697) + + test: make ansible test work across older versions (#1691) + + Networkd multi-address support/fix (#1685) [Teodor Garzdin] + + make: drop broken targets (#1688) + + net: Passthough v2 netconfigs in netplan systems (#1650) + + NM ipv6 connection does not work on Azure and Openstack (#1616) + [Emanuele Giuseppe Esposito] + + Fix check_format_tip (#1679) + + DataSourceVMware: fix var use before init (#1674) [Andrew Kutz] + + rpm/copr: ensure RPM represents new clean.d dir artifacts (#1680) + + test: avoid centos leaked check of /etc/yum.repos.d/epel-testing.repo + (#1676) + + Release 22.3 (#1662) + + sources: obj.pkl cache should be written anyime get_data is run + (#1669) + + schema: drop release number from version file (#1664) + + pycloudlib: bump to quiet azure HTTP info logs (#1668) + + test: fix wireguard integration tests (#1666) + + Github is deprecating the 18.04 runner starting 12.1 (#1665) + + integration tests: Ensure one setup for all tests (#1661) + + tests: ansible test fixes (#1660) + + Prevent concurrency issue in test_webhook_hander.py (#1658) + + Workaround net_setup_link race with udev (#1655) + + test: drop erroneous lxd assertion, verify command succeeded (#1657) + + Fix Chrony usage on Centos Stream (#1648) + [Sven Haardiek] + + sources/azure: handle network unreachable errors for saveable PPS + (#1642) [Chris Patterson] + + Return cc_set_hostname to PER_INSTANCE frequency (#1651) + + test: Collect integration test time by default (#1638) + + test: Drop forced package install hack in lxd integration test + (#1649) + + schema: Resolve user-data if --system given (#1644) + [Alberto Contreras] + + test: use fake filesystem to avoid file removal (#1647) + [Alberto Contreras] + + tox: Fix tip-flake8 and tip-mypy (#1635) [Alberto Contreras] + + config: Add wireguard config module (#1570) + [Fabian Lichtenegger-Lukas] + + tests: can run without azure-cli, tests expect inactive ansible + (#1643) + + typing: Type UrlResponse.contents (#1633) [Alberto Contreras] + + testing: fix references to `DEPRECATED.` (#1641) + [Alberto Contreras] + + ssh_util: Handle sshd_config.d folder + [Alberto Contreras] + + schema: Enable deprecations in cc_update_etc_hosts (#1631) + [Alberto Contreras] + + Add Ansible Config Module (#1579) + + util: Support Idle process state in get_proc_ppid() (#1637) + + schema: Enable deprecations in cc_growpart (#1628) + [Alberto Contreras] + + schema: Enable deprecations in cc_users_groups (#1627) + [Alberto Contreras] + + util: Fix error path and parsing in get_proc_ppid() + + main: avoid downloading full contents cmdline urls (#1606) + + schema: Enable deprecations in cc_scripts_vendor (#1629) + [Alberto Contreras] + + schema: Enable deprecations in cc_set_passwords (#1630) + [Alberto Contreras] + + sources/azure: add experimental support for preprovisioned os disks + (#1622) [Chris Patterson] + + Remove configobj a_to_u calls (#1632) [Stefano Rivera] + + cc_debug: Drop this module (#1614) [Alberto Contreras] + + schema: add aggregate descriptions in anyOf/oneOf (#1636) + + testing: migrate test_sshutil to pytest (#1617) [Alberto Contreras] + + testing: Fix test_ca_certs integration test (#1626) + [Alberto Contreras] + + testing: add support for pycloudlib's pro images (#1604) + [Alberto Contreras] + + testing: migrate test_cc_set_passwords to pytest (#1615) + [Alberto Contreras] + + network: add system_info network activator cloud.cfg overrides + (#1619) + + docs: Align git remotes with uss-tableflip setup (#1624) + [Alberto Contreras] + + testing: cover active config module checks (#1609) + [Alberto Contreras] + + lxd: lvm avoid thinpool when kernel module absent + + lxd: enable MTU configuration in cloud-init + + doc: pin doc8 to last passing version + + cc_set_passwords fixes (#1590) + + Modernise importer.py and type ModuleDetails (#1605) + [Alberto Contreras] + + config: Def activate_by_schema_keys for t-z (#1613) + [Alberto Contreras] + + config: define activate_by_schema_keys for p-r mods (#1611) + [Alberto Contreras] + + clean: add param to remove /etc/machine-id for golden image + creation + + config: define `activate_by_schema_keys` for a-f mods (#1608) + [Alberto Contreras] + + config: define activate_by_schema_keys for s mods (#1612) + [Alberto Contreras] + + sources/azure: reorganize tests for network config (#1586) + [Chris Patterson] + + config: Define activate_by_schema_keys for g-n mods (#1610) + [Alberto Contreras] + + meta-schema: add infra to skip inapplicable modules + [Alberto Contreras] + + sources/azure: don't set cfg["password"] for default user pw + (#1592) [Chris Patterson] + + schema: activate grub-dpkg deprecations (#1600) [Alberto Contreras] + + docs: clarify user password purposes (#1593) + + cc_lxd: Add btrfs and lvm lxd storage options (SC-1026) (#1585) + + archlinux: Fix distro naming[1] (#1601) [Kristian Klausen] + + cc_ubuntu_autoinstall: support live-installer autoinstall config + + clean: allow third party cleanup scripts in /etc/cloud/clean.d + (#1581) + + sources/azure: refactor chassis asset tag handling (#1574) + [Chris Patterson] + + Add "netcho" as contributor (#1591) [Kaloyan Kotlarski] + + testing: drop impish support (#1596) [Alberto Contreras] + + black: fix missed formatting issue which landed in main (#1594) + + bsd: Don't assume that root user is in root group (#1587) + + docs: Fix comment typo regarding use of packages (#1582) + [Peter Mescalchin] + + Update govc command in VMWare walkthrough (#1576) [manioo8] + + Update .github-cla-signers (#1588) [Daniel Mullins] + + Rename the openmandriva user to omv (#1575) + [Bernhard Rosenkraenzer] + + sources/azure: increase read-timeout to 60 seconds for wireserver + (#1571) [Chris Patterson] + + Resource leak cleanup (#1556) + + testing: remove appereances of FakeCloud (#1584) + [Alberto Contreras] + + Fix expire passwords for hashed passwords (#1577) + [Sadegh Hayeri] + + mounts: fix suggested_swapsize for > 64GB hosts (#1569) + [Steven Stallion] + + Update chpasswd schema to deprecate password parsing (#1517) + + tox: Remove entries from default envlist (#1578) + + tests: add test for parsing static dns for existing devices (#1557) + [Jonas Konrad] + + testing: port cc_ubuntu_advantage test to pytest (#1559) + [Alberto Contreras] + + Schema deprecation handling (#1549) [Alberto Contreras] + + Enable pytest to run in parallel (#1568) + + sources/azure: refactor ovf-env.xml parsing (#1550) + [Chris Patterson] + + schema: Force stricter validation (#1547) + + ubuntu advantage config: http_proxy, https_proxy (#1512) + [Fabian Lichtenegger-Lukas] + + travis: Upgrade dist to focal [Alberto Contreras] + + net: fix interface matching support (#1552) + + Fuzz testing jsonchema (#1499) [Alberto Contreras] + + testing: Wait for changed boot-id in test_status.py (#1548) + + CI: Fix GH pinned-format jobs (#1558) [Alberto Contreras] + + Typo fix (#1560) [Jaime Hablutzel] + + tests: mock dns lookup that causes long timeouts (#1555) + + tox: add unpinned env for do_format and check_format (#1554) + + cc_ssh_import_id: Substitute deprecated warn (#1553) + [Alberto Contreras] + + Remove schema errors from log (#1551) + + Update WebHookHandler to run as background thread (SC-456) (#1491) + + testing: Don't run custom cloud dir test on Bionic (#1542) + + bash completion: update schema command (#1543) + + CI: add non-blocking run against the linters tip versions (#1531) + [Paride Legovini] + + Change groups within the users schema to support lists and strings + (#1545) [RedKrieg] + + make it clear which username should go in the contributing doc + (#1546) + + Pin setuptools for Travis (SC-1136) (#1540) + + Fix LXD datasource crawl when BOOT enabled (#1537) + + testing: Fix wrong path in dual stack test (#1538) + + cloud-config: honor cloud_dir setting (#1523) + [Alberto Contreras] + + Add python3-debconf to pkg-deps.json Build-Depends (#1535) + [Alberto Contreras] + + redhat spec: udev/rules.d lives under /usr/lib on rhel-based + systems (#1536) + + tests/azure: add test coverage for DisableSshPasswordAuthentication + (#1534) [Chris Patterson] + + summary: Add david-caro to the cla signers (#1527) [David Caro] + + Add support for OpenMandriva (https://openmandriva.org/) (#1520) + [Bernhard Rosenkraenzer] + + tests/azure: refactor ovf creation (#1533) [Chris Patterson] + + Improve DataSourceOVF error reporting when script disabled (#1525) + [rong] + + tox: integration-tests-jenkins: softfail if only some test failed + (#1528) [Paride Legovini] + + CI: drop linters from Travis CI (moved to GH Actions) (#1530) + [Paride Legovini] + + sources/azure: remove unused encoding support for customdata + (#1526) [Chris Patterson] + + sources/azure: remove unused metadata captured when parsing ovf + (#1524) [Chris Patterson] + + sources/azure: remove dscfg parsing from ovf-env.xml (#1522) + [Chris Patterson] + + Remove extra space from ec2 dual stack crawl message (#1521) + + tests/azure: use namespaces in generated ovf-env.xml documents + (#1519) [Chris Patterson] + + setup.py: adjust udev/rules default path (#1513) + [Emanuele Giuseppe Esposito] + + Add python3-deconf dependency (#1506) [Alberto Contreras] + + Change match macadress param for network v2 config (#1518) + [Henrique Caricatti Capozzi] + + sources/azure: remove unused userdata property from ovf (#1516) + [Chris Patterson] + + sources/azure: minor refactoring to network config generation + (#1497) [Chris Patterson] + + net: Implement link-local ephemeral ipv6 + + Rename function to avoid confusion (#1501) + + Fix cc_phone_home requiring 'tries' (#1500) + + datasources: replace networking functions with stdlib and + cloudinit.net code + + Remove xenial references (#1472) [Alberto Contreras] + + Oracle ds changes (#1474) + + improve runcmd docs (#1498) + + add 3.11-dev to Travis CI (#1493) + + Only run github actions on pull request (#1496) + + Fix integration test client creation (#1494) [Alberto Contreras] + + tox: add link checker environment, fix links (#1480) + + cc_ubuntu_advantage: Fix doc (#1487) [Alberto Contreras] + + cc_yum_add_repo: Fix repo id canonicalization (#1489) + [Alberto Contreras] + + Add linitio as contributor in the project (#1488) [Kevin Allioli] + + net-convert: use yaml.dump for debugging python NetworkState obj + (#1484) + + test_schema: no relative $ref URLs, replace $ref with local path + (#1486) + + cc_set_hostname: do not write "localhost" when no hostname is given + (#1453) [Emanuele Giuseppe Esposito] + + Update .github-cla-signers (#1478) [rong] + + schema: write_files defaults, versions $ref full URL and add vscode + (#1479) + + docs: fix external links, add one more to the list (#1477) + + doc: Document how to change module frequency (#1481) + + tests: bump pycloudlib (#1482) + + tests: bump pycloudlib pinned commit for kinetic Azure (#1476) + + testing: fix test_status.py (#1475) + + integration tests: If KEEP_INSTANCE = True, log IP (#1473) + + Drop mypy excluded files (#1454) [Alberto Contreras] + + Docs additions (#1470) + + Add "formatting tests" to Github Actions + + Remove unused arguments in function signature (#1471) + + Changelog: correct errant classification of LP issues as GH (#1464) + + Use Network-Manager and Netplan as default renderers for RHEL and + Fedora (#1465) [Emanuele Giuseppe Esposito] + + -- Brett Holman Tue, 30 Aug 2022 15:14:08 -0600 + +cloud-init (22.2-0ubuntu1~20.04.3) focal; urgency=medium + + * d/cloud-init.postinst: redact previously leaked schema errors from logs + * Remove schema errors from log (LP: #1978422) (CVE-2022-2084) -- James Falcon Wed, 15 Jun 2022 11:34:44 -0500 @@ -3494,7 +3777,7 @@ cloud-init (0.7.8-34-ga1cdebd-0ubuntu1) zesty; urgency=medium * New upstream snapshot. - net/cmdline: Further adjustments to ipv6 support [LaMont Jones] - (LP: #1621615) + (LP: #1621615) - Add coverage dependency to bddeb to fix package build. - doc: improve HACKING.rst file - dmidecode: Allow dmidecode to be used on aarch64 [Robert Schweikert] diff --git a/debian/cloud-init.lintian-overrides b/debian/cloud-init.lintian-overrides index 45037735b..952adacc1 100644 --- a/debian/cloud-init.lintian-overrides +++ b/debian/cloud-init.lintian-overrides @@ -3,3 +3,6 @@ cloud-init binary: systemd-service-file-refers-to-unusual-wantedby-target lib/sy cloud-init binary: systemd-service-file-refers-to-unusual-wantedby-target lib/systemd/system/cloud-init-local.service cloud-init.target cloud-init binary: systemd-service-file-refers-to-unusual-wantedby-target lib/systemd/system/cloud-init.service cloud-init.target cloud-init binary: package-supports-alternative-init-but-no-init.d-script + +# Legit use of dpkg-database +cloud-init: uses-dpkg-database-directly usr/lib/python3/dist-packages/cloudinit/distros/debian.py diff --git a/debian/cloud-init.postinst b/debian/cloud-init.postinst index 1e320176d..de4872417 100644 --- a/debian/cloud-init.postinst +++ b/debian/cloud-init.postinst @@ -168,7 +168,7 @@ fix_1336855() { [ -r /proc/cmdline ] || return 0 # Don't do anything unless we have grub - command -v grub-install > /dev/null 2>&1 || return 0 + command -v grub-install > /dev/null || return 0 # First, identify the kernel device for the parent. for parm in $(cat /proc/cmdline); do @@ -321,7 +321,7 @@ fix_lp1889555() { [ -f /var/lib/cloud/instance/sem/config_grub_dpkg ] || return 0 # Don't do anything unless we have grub - [ -x /usr/sbin/grub-install ] || return 0 + command -v grub-install > /dev/null || return 0 # Make sure that we are not chrooted. [ "$(stat -c %d:%i /)" != "$(stat -c %d:%i /proc/1/root/.)" ] && return 0 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control index 555e8cea8..db68d4ce4 100644 --- a/debian/control +++ b/debian/control @@ -3,13 +3,14 @@ Section: admin Priority: optional Homepage: https://cloud-init.io/ Maintainer: Ubuntu Developers -Build-Depends: debhelper (>= 9.20160709), +Build-Depends: debhelper-compat (= 10), dh-python, iproute2, pep8, po-debconf, python3, python3-configobj, + python3-debconf, python3-httpretty, python3-jinja2, python3-jsonpatch, @@ -38,6 +39,7 @@ Depends: cloud-guest-utils | cloud-utils, netplan.io, procps, python3, + python3-debconf, python3-netifaces, python3-requests, python3-serial, diff --git a/debian/patches/cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500 b/debian/patches/cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500 deleted file mode 100644 index 225ab50e3..000000000 --- a/debian/patches/cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500 +++ /dev/null @@ -1,158 +0,0 @@ -From a2e6273872c2cafa73530e6c73178c2ebfc2beba Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Thu, 9 Jun 2022 14:59:52 -0500 -Subject: [PATCH] Fix cc_phone_home requiring 'tries' (#1500) - -As part of dee8231, the code was updated to use a ValueError rather than -catching the top level Exception. This should have also included -catching TypeError. - -Additionally, add unit tests for module as it effectively had none. - -LP: #1977952 ---- - cloudinit/config/cc_phone_home.py | 13 +-- - tests/unittests/config/test_cc_phone_home.py | 87 ++++++++++++++++++++ - 2 files changed, 94 insertions(+), 6 deletions(-) - -Index: cloud-init/cloudinit/config/cc_phone_home.py -=================================================================== ---- cloud-init.orig/cloudinit/config/cc_phone_home.py -+++ cloud-init/cloudinit/config/cc_phone_home.py -@@ -130,7 +130,7 @@ def handle(name, cfg, cloud, log, args): - tries = ph_cfg.get("tries") - try: - tries = int(tries) # type: ignore -- except ValueError: -+ except (ValueError, TypeError): - tries = 10 - util.logexc( - log, -@@ -141,10 +141,11 @@ def handle(name, cfg, cloud, log, args): - if post_list == "all": - post_list = POST_LIST_ALL - -- all_keys = {} -- all_keys["instance_id"] = cloud.get_instance_id() -- all_keys["hostname"] = cloud.get_hostname() -- all_keys["fqdn"] = cloud.get_hostname(fqdn=True) -+ all_keys = { -+ "instance_id": cloud.get_instance_id(), -+ "hostname": cloud.get_hostname(), -+ "fqdn": cloud.get_hostname(fqdn=True), -+ } - - pubkeys = { - "pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub", -@@ -190,7 +191,7 @@ def handle(name, cfg, cloud, log, args): - url_helper.read_file_or_url( - url, - data=real_submit_keys, -- retries=tries, -+ retries=tries - 1, - sec_between=3, - ssl_details=util.fetch_ssl_details(cloud.paths), - ) -Index: cloud-init/tests/unittests/config/test_cc_phone_home.py -=================================================================== ---- cloud-init.orig/tests/unittests/config/test_cc_phone_home.py -+++ cloud-init/tests/unittests/config/test_cc_phone_home.py -@@ -1,11 +1,98 @@ -+import logging -+from functools import partial -+from itertools import count -+from unittest import mock -+ - import pytest - -+from cloudinit.config.cc_phone_home import POST_LIST_ALL, handle - from cloudinit.config.schema import ( - SchemaValidationError, - get_schema, - validate_cloudconfig_schema, - ) - from tests.unittests.helpers import skipUnlessJsonSchema -+from tests.unittests.util import get_cloud -+ -+LOG = logging.getLogger("TestNoConfig") -+phone_home = partial(handle, name="test", cloud=get_cloud(), log=LOG, args=[]) -+ -+ -+@pytest.fixture(autouse=True) -+def common_mocks(mocker): -+ mocker.patch("cloudinit.util.load_file", side_effect=count()) -+ -+ -+@mock.patch("cloudinit.url_helper.readurl") -+class TestPhoneHome: -+ def test_default_call(self, m_readurl): -+ cfg = {"phone_home": {"url": "myurl"}} -+ phone_home(cfg=cfg) -+ assert m_readurl.call_args == mock.call( -+ "myurl", -+ data={ -+ "pub_key_dsa": "0", -+ "pub_key_rsa": "1", -+ "pub_key_ecdsa": "2", -+ "pub_key_ed25519": "3", -+ "instance_id": "iid-datasource-none", -+ "hostname": "hostname", -+ "fqdn": "hostname", -+ }, -+ retries=9, -+ sec_between=3, -+ ssl_details={}, -+ ) -+ -+ def test_no_url(self, m_readurl, caplog): -+ cfg = {"phone_home": {}} -+ phone_home(cfg=cfg) -+ assert "Skipping module named" in caplog.text -+ assert m_readurl.call_count == 0 -+ -+ @pytest.mark.parametrize( -+ "tries, expected_retries", -+ [ -+ (-1, -2), -+ (0, -1), -+ (1, 0), -+ (2, 1), -+ ("2", 1), -+ ("two", 9), -+ (None, 9), -+ ({}, 9), -+ ], -+ ) -+ def test_tries(self, m_readurl, tries, expected_retries, caplog): -+ cfg = {"phone_home": {"url": "dontcare"}} -+ if tries is not None: -+ cfg["phone_home"]["tries"] = tries -+ phone_home(cfg=cfg) -+ assert m_readurl.call_args[1]["retries"] == expected_retries -+ -+ def test_post_all(self, m_readurl): -+ cfg = {"phone_home": {"url": "test", "post": "all"}} -+ phone_home(cfg=cfg) -+ for key in POST_LIST_ALL: -+ assert key in m_readurl.call_args[1]["data"] -+ -+ def test_custom_post_list(self, m_readurl): -+ post_list = ["pub_key_rsa, hostname"] -+ cfg = {"phone_home": {"url": "test", "post": post_list}} -+ phone_home(cfg=cfg) -+ for key in post_list: -+ assert key in m_readurl.call_args[1]["data"] -+ assert len(m_readurl.call_args[1]["data"]) == len(post_list) -+ -+ def test_invalid_post(self, m_readurl, caplog): -+ post_list = ["spam", "hostname"] -+ cfg = {"phone_home": {"url": "test", "post": post_list}} -+ phone_home(cfg=cfg) -+ assert "hostname" in m_readurl.call_args[1]["data"] -+ assert m_readurl.call_args[1]["data"]["spam"] == "N/A" -+ assert ( -+ "spam from 'post' configuration list not available" in caplog.text -+ ) - - - class TestPhoneHomeSchema: diff --git a/debian/patches/cpick-b0534cbf-Remove-schema-errors-from-log b/debian/patches/cpick-b0534cbf-Remove-schema-errors-from-log deleted file mode 100644 index 01e886d7b..000000000 --- a/debian/patches/cpick-b0534cbf-Remove-schema-errors-from-log +++ /dev/null @@ -1,148 +0,0 @@ -From b0534cbf05221b141ebd2edb5a71e94742b369a3 Mon Sep 17 00:00:00 2001 -From: James Falcon -Date: Tue, 14 Jun 2022 06:24:40 -0500 -Subject: [PATCH] Remove schema errors from log - -When schema errors are encountered, the section of userdata in question -gets printed to the cloud-init log. As this could contain sensitive -data, so log a generic warning instead and redirect user to run -`cloud-init schema --system` as root. - -LP: #1978422 ---- - cloudinit/cmd/main.py | 4 +++- - cloudinit/config/schema.py | 15 +++++++++++--- - tests/integration_tests/modules/test_cli.py | 20 ++++++++++++------ - tests/unittests/config/test_schema.py | 23 ++++++++++++++++++++- - 4 files changed, 51 insertions(+), 11 deletions(-) - ---- a/cloudinit/cmd/main.py -+++ b/cloudinit/cmd/main.py -@@ -454,7 +454,9 @@ def main_init(name, args): - - # Validate user-data adheres to schema definition - if os.path.exists(init.paths.get_ipath_cur("userdata_raw")): -- validate_cloudconfig_schema(config=init.cfg, strict=False) -+ validate_cloudconfig_schema( -+ config=init.cfg, strict=False, log_details=False -+ ) - else: - LOG.debug("Skipping user-data validation. No user-data found.") - ---- a/cloudinit/config/schema.py -+++ b/cloudinit/config/schema.py -@@ -196,6 +196,7 @@ def validate_cloudconfig_schema( - schema: dict = None, - strict: bool = False, - strict_metaschema: bool = False, -+ log_details: bool = True, - ): - """Validate provided config meets the schema definition. - -@@ -208,6 +209,9 @@ def validate_cloudconfig_schema( - logging warnings. - @param strict_metaschema: Boolean, when True validates schema using strict - metaschema definition at runtime (currently unused) -+ @param log_details: Boolean, when True logs details of validation errors. -+ If there are concerns about logging sensitive userdata, this should -+ be set to False. - - @raises: SchemaValidationError when provided config does not validate - against the provided schema. -@@ -232,12 +236,17 @@ def validate_cloudconfig_schema( - errors += ((path, error.message),) - if errors: - if strict: -+ # This could output/log sensitive data - raise SchemaValidationError(errors) -- else: -+ if log_details: - messages = ["{0}: {1}".format(k, msg) for k, msg in errors] -- LOG.warning( -- "Invalid cloud-config provided:\n%s", "\n".join(messages) -+ details = "\n" + "\n".join(messages) -+ else: -+ details = ( -+ "Please run 'sudo cloud-init schema --system' to " -+ "see the schema errors." - ) -+ LOG.warning("Invalid cloud-config provided: %s", details) - - - def annotated_cloudconfig_file( ---- a/tests/integration_tests/modules/test_cli.py -+++ b/tests/integration_tests/modules/test_cli.py -@@ -18,11 +18,18 @@ runcmd: - - echo 'hi' > /var/tmp/test - """ - -+# The '-' in 'hashed-password' fails schema validation - INVALID_USER_DATA_SCHEMA = """\ - #cloud-config --updates: -- notnetwork: -1 --apt_pipelining: bogus -+users: -+ - default -+ - name: newsuper -+ gecos: Big Stuff -+ groups: users, admin -+ sudo: ALL=(ALL) NOPASSWD:ALL -+ hashed-password: asdfasdf -+ shell: /bin/bash -+ lock_passwd: true - """ - - -@@ -69,11 +76,12 @@ def test_invalid_userdata_schema(client: - assert result.ok - log = client.read_from_file("/var/log/cloud-init.log") - warning = ( -- "[WARNING]: Invalid cloud-config provided:\napt_pipelining: 'bogus'" -- " is not valid under any of the given schemas\nupdates: Additional" -- " properties are not allowed ('notnetwork' was unexpected)" -+ "[WARNING]: Invalid cloud-config provided: Please run " -+ "'sudo cloud-init schema --system' to see the schema errors." - ) - assert warning in log -+ assert "asdfasdf" not in log -+ - result = client.execute("cloud-init status --long") - if not result.ok: - raise AssertionError( ---- a/tests/unittests/config/test_schema.py -+++ b/tests/unittests/config/test_schema.py -@@ -304,11 +304,32 @@ class TestValidateCloudConfigSchema: - assert "cloudinit.config.schema" == module - assert logging.WARNING == log_level - assert ( -- "Invalid cloud-config provided:\np1: -1 is not of type 'string'" -+ "Invalid cloud-config provided: \np1: -1 is not of type 'string'" - == log_msg - ) - - @skipUnlessJsonSchema() -+ def test_validateconfig_schema_sensitive(self, caplog): -+ """When log_details=False, ensure details are omitted""" -+ schema = { -+ "properties": {"hashed_password": {"type": "string"}}, -+ "additionalProperties": False, -+ } -+ validate_cloudconfig_schema( -+ {"hashed-password": "secret"}, -+ schema, -+ strict=False, -+ log_details=False, -+ ) -+ [(module, log_level, log_msg)] = caplog.record_tuples -+ assert "cloudinit.config.schema" == module -+ assert logging.WARNING == log_level -+ assert ( -+ "Invalid cloud-config provided: Please run 'sudo cloud-init " -+ "schema --system' to see the schema errors." == log_msg -+ ) -+ -+ @skipUnlessJsonSchema() - def test_validateconfig_schema_emits_warning_on_missing_jsonschema( - self, caplog - ): diff --git a/debian/patches/expire-on-hashed-users.patch b/debian/patches/expire-on-hashed-users.patch new file mode 100644 index 000000000..062574d98 --- /dev/null +++ b/debian/patches/expire-on-hashed-users.patch @@ -0,0 +1,87 @@ +Description: Ensure password expire doesn't apply to hashed users +Author: james.falcon@canonical.com +Origin: backport +Forwarded: not-needed +Last-Update: 2022-08-12 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/cloudinit/features.py ++++ b/cloudinit/features.py +@@ -50,7 +50,7 @@ directives in cloud-config. + """ + + +-EXPIRE_APPLIES_TO_HASHED_USERS = True ++EXPIRE_APPLIES_TO_HASHED_USERS = False + """ + If ``EXPIRE_APPLIES_TO_HASHED_USERS`` is True, then when expire is set true + in cc_set_passwords, hashed passwords will be expired. Previous to 22.3, +--- a/tests/unittests/config/test_cc_set_passwords.py ++++ b/tests/unittests/config/test_cc_set_passwords.py +@@ -330,58 +330,6 @@ class TestSetPasswordsHandle: + @pytest.mark.parametrize( + "user_cfg", + [ +- { +- "list": [ +- "ubuntu:passw0rd", +- "sadegh:$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", +- ] +- }, +- { +- "users": [ +- { +- "name": "ubuntu", +- "password": "passw0rd", +- "type": "text", +- }, +- { +- "name": "sadegh", +- "password": "$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", # noqa: E501 +- }, +- ] +- }, +- ], +- ) +- def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( +- self, user_cfg, mocker +- ): +- """BSD don't use chpasswd""" +- mocker.patch(f"{MODPATH}util.is_BSD", return_value=True) +- m_subp = mocker.patch(f"{MODPATH}subp.subp") +- cloud = get_cloud(distro="freebsd") +- cfg = {"chpasswd": user_cfg} +- with mock.patch.object( +- cloud.distro, "uses_systemd", return_value=False +- ): +- setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) +- assert [ +- mock.call( +- ["pw", "usermod", "ubuntu", "-h", "0"], +- data="passw0rd", +- logstring="chpasswd for ubuntu", +- ), +- mock.call( +- ["pw", "usermod", "sadegh", "-H", "0"], +- data="$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", +- logstring="chpasswd for sadegh", +- ), +- mock.call(["pw", "usermod", "ubuntu", "-p", "01-Jan-1970"]), +- mock.call(["pw", "usermod", "sadegh", "-p", "01-Jan-1970"]), +- mock.call(["service", "sshd", "status"], capture=True), +- ] == m_subp.call_args_list +- +- @pytest.mark.parametrize( +- "user_cfg", +- [ + {"expire": "false", "list": ["root:R", "ubuntu:RANDOM"]}, + { + "expire": "false", +@@ -646,6 +594,7 @@ expire_cases = [ + class TestExpire: + @pytest.mark.parametrize("cfg", expire_cases) + def test_expire(self, cfg, mocker, caplog): ++ features.EXPIRE_APPLIES_TO_HASHED_USERS = True + cloud = get_cloud() + mocker.patch(f"{MODPATH}subp.subp") + mocker.patch.object(cloud.distro, "chpasswd") diff --git a/debian/patches/series b/debian/patches/series index 739b2c20b..7accf181e 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,3 +1,2 @@ retain-apt-partner-pocket.patch -cpick-a2e62738-Fix-cc_phone_home-requiring-tries-1500 -cpick-b0534cbf-Remove-schema-errors-from-log +expire-on-hashed-users.patch diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides new file mode 100644 index 000000000..5b8bf5cb8 --- /dev/null +++ b/debian/source/lintian-overrides @@ -0,0 +1,2 @@ +# Silence lintian bug: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1014584 +cloud-init source: binary-nmu-debian-revision-in-source diff --git a/doc-requirements.txt b/doc-requirements.txt index 38c943e1d..3207e6c60 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,4 +1,5 @@ -doc8 +# doc8 1.0.0 depends on docutils 0.18.1 or later which added Node.findall() +doc8==0.11.2 m2r2 sphinx==4.3.0 sphinx_rtd_theme==1.0.0 diff --git a/doc/examples/cloud-config-ansible.txt b/doc/examples/cloud-config-ansible.txt new file mode 100644 index 000000000..a3e7c273c --- /dev/null +++ b/doc/examples/cloud-config-ansible.txt @@ -0,0 +1,16 @@ +#cloud-config +version: v1 +packages_update: true +packages_upgrade: true + +# if you're already installing other packages, you may +# wish to manually install ansible to avoid multiple calls +# to your package manager +packages: + - ansible + - git +ansible: + install-method: pip + pull: + url: "https://github.com/holmanb/vmboot.git" + playbook-name: ubuntu.yml diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt index efeae6250..dd6a0f6aa 100644 --- a/doc/examples/cloud-config-apt.txt +++ b/doc/examples/cloud-config-apt.txt @@ -35,7 +35,7 @@ apt_pipelining: False # # Default: none # -# if packages are specified, this package_update will be set to true +# if packages are specified, then package_update will be set to true packages: ['pastebinit'] diff --git a/doc/examples/cloud-config-install-packages.txt b/doc/examples/cloud-config-install-packages.txt index 8bd9b74f0..ea1e57432 100644 --- a/doc/examples/cloud-config-install-packages.txt +++ b/doc/examples/cloud-config-install-packages.txt @@ -4,7 +4,7 @@ # # Default: none # -# if packages are specified, this package_update will be set to true +# if packages are specified, then package_update will be set to true # # packages may be supplied as a single package name or as a list # with the format [, ] wherein the specific diff --git a/doc/examples/cloud-config-lxd.txt b/doc/examples/cloud-config-lxd.txt index e96f314b3..512b3f08a 100644 --- a/doc/examples/cloud-config-lxd.txt +++ b/doc/examples/cloud-config-lxd.txt @@ -7,7 +7,7 @@ # init: dict of options for lxd init, see 'man lxd' # network_address: address for lxd to listen on # network_port: port for lxd to listen on -# storage_backend: either 'zfs' or 'dir' +# storage_backend: 'zfs', 'dir', 'lvm', or 'btrfs' # storage_create_device: device based storage using specified device # storage_create_loop: set up loop based storage with size in GB # storage_pool: name of storage pool to use or create diff --git a/doc/examples/cloud-config-reporting.txt b/doc/examples/cloud-config-reporting.txt index 80bde3037..a4ebabfdb 100644 --- a/doc/examples/cloud-config-reporting.txt +++ b/doc/examples/cloud-config-reporting.txt @@ -2,7 +2,6 @@ ## ## The following sets up 2 reporting end points. ## A 'webhook' and a 'log' type. -## It also disables the built in default 'log' reporting: smtest: type: webhook @@ -14,4 +13,3 @@ reporting: smlogger: type: log level: WARN - log: null diff --git a/doc/examples/cloud-config-wireguard.txt b/doc/examples/cloud-config-wireguard.txt new file mode 100644 index 000000000..11920f24c --- /dev/null +++ b/doc/examples/cloud-config-wireguard.txt @@ -0,0 +1,29 @@ +#cloud-config +# vim: syntax=yaml +# +# This is the configuration syntax that the wireguard module +# will know how to understand. +# +# +wireguard: + # All wireguard interfaces that should be created. Every interface will be named + # after `name` parameter and config will be written to a file under `config_path`. + # `content` parameter should be set with a valid Wireguard configuration. + interfaces: + - name: wg0 + config_path: /etc/wireguard/wg0.conf + content: | + [Interface] + PrivateKey = + Address =
+ [Peer] + PublicKey = + Endpoint = : + AllowedIPs = , , ... + # The idea behind readiness probes is to ensure Wireguard connectivity before continuing + # the cloud-init process. This could be useful if you need access to specific services like + # an internal APT Repository Server (e.g Landscape) to install/update packages. + readinessprobe: + - 'systemctl restart service' + - 'curl https://webhook.endpoint/example' + - 'nc -zv apt-server-fqdn 443' diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 177c56000..7f4ded8c5 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -428,10 +428,15 @@ syslog_fix_perms: syslog:root # to set hashed password, here account 'user3' has a password it set to # 'cloud-init', hashed with SHA-256: # chpasswd: -# list: | -# user1:password1 -# user2:RANDOM -# user3:$5$eriogqzq$Dg7PxHsKGzziuEGkZgkLvacjuEFeljJ.rLf.hZqKQLA +# users: +# - name: user1 +# password: password1 +# type: text +# - user2 +# type: RANDOM +# - user3 +# password: $5$eriogqzq$Dg7PxHsKGzziuEGkZgkLvacjuEFeljJ.rLf.hZqKQLA +# type: hash # expire: True # ssh_pwauth: [ True, False, "" or "unchanged" ] # diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index d8ca9d16e..aa9782377 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -18,7 +18,7 @@ Cloud-init has support across all major Linux distributions, FreeBSD, NetBSD, OpenBSD and DragonFlyBSD: - Alpine Linux -- ArchLinux +- Arch Linux - Debian - DragonFlyBSD - Fedora diff --git a/doc/rtd/topics/boot.rst b/doc/rtd/topics/boot.rst index e06637607..db6621a7a 100644 --- a/doc/rtd/topics/boot.rst +++ b/doc/rtd/topics/boot.rst @@ -149,7 +149,7 @@ accustomed to running after logging into a system should run correctly here. Things that run here include: * package installations - * configuration management plugins (puppet, chef, salt-minion) + * configuration management plugins (ansible, puppet, chef, salt-minion) * user-defined scripts (i.e. shell scripts passed as user-data) For scripts external to cloud-init looking to wait until cloud-init is diff --git a/doc/rtd/topics/bugs.rst b/doc/rtd/topics/bugs.rst index ee3828dea..c66048e2b 100644 --- a/doc/rtd/topics/bugs.rst +++ b/doc/rtd/topics/bugs.rst @@ -88,11 +88,11 @@ SUSE & openSUSE To file a bug against the SuSE packages of cloud-init please use the `SUSE bugzilla`_. -Arch ----- +Arch Linux +---------- To file a bug against the Arch package of cloud-init please use the -`Arch Linux Bugtracker`_. See the `Arch bug reporting wiki`_ for more +`Arch Linux Bugtracker`_. See the `Arch Linux bug reporting wiki`_ for more details. .. _Create a Launchpad account: https://help.launchpad.net/YourAccount/NewAccount @@ -103,6 +103,6 @@ details. .. _Red Hat bugzilla: https://bugzilla.redhat.com/ .. _SUSE bugzilla: https://bugzilla.suse.com/index.cgi .. _Arch Linux Bugtracker: https://bugs.archlinux.org/ -.. _Arch bug reporting wiki: https://wiki.archlinux.org/index.php/Bug_reporting_guidelines +.. _Arch Linux bug reporting wiki: https://wiki.archlinux.org/index.php/Bug_reporting_guidelines .. vi: textwidth=79 diff --git a/doc/rtd/topics/cli.rst b/doc/rtd/topics/cli.rst index 2e209bb44..1a5f5e2da 100644 --- a/doc/rtd/topics/cli.rst +++ b/doc/rtd/topics/cli.rst @@ -67,6 +67,9 @@ instance. On reboot, cloud-init will re-run all stages as it did on first boot. * ``--logs``: optionally remove all cloud-init log files in ``/var/log/`` * ``--reboot``: reboot the system after removing artifacts +* ``--machine-id``: Remove ``/etc/machine-id`` on this image. Best practice + when cloning a golden image to ensure that the next boot of that image + auto-generates an unique machine ID. `More details on machine-id`_. .. _cli_collect_logs: @@ -323,3 +326,5 @@ the currently running modules, as well as when it is done. time: Wed, 17 Jan 2018 20:41:59 +0000 detail: DataSourceNoCloud [seed=/var/lib/cloud/seed/nocloud-net][dsmode=net] + +.. _More details on machine-id: https://www.freedesktop.org/software/systemd/man/machine-id.html diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst index 77232269b..d30e1bb61 100644 --- a/doc/rtd/topics/datasources/ec2.rst +++ b/doc/rtd/topics/datasources/ec2.rst @@ -89,7 +89,8 @@ The settings that may be configured are: * **metadata_urls**: This list of urls will be searched for an EC2 metadata service. The first entry that successfully returns a 200 response for //meta-data/instance-id will be selected. - (default: ['http://169.254.169.254', 'http://instance-data:8773']). + (default: ['http://169.254.169.254', 'http://[fd00:ec2::254]', + 'http://instance-data:8773']). * **max_wait**: the maximum amount of clock time in seconds that should be spent searching metadata_urls. A value less than zero will result in only one request being made, to the first in the list. (default: 120) diff --git a/doc/rtd/topics/datasources/gce.rst b/doc/rtd/topics/datasources/gce.rst index 70aefea28..3aeb9afc0 100644 --- a/doc/rtd/topics/datasources/gce.rst +++ b/doc/rtd/topics/datasources/gce.rst @@ -37,6 +37,6 @@ An example configuration with the default values is provided below: retries: 5 sec_between_retries: 1 -.. _GCE metadata docs: https://cloud.google.com/compute/docs/storing-retrieving-metadata#querying +.. _GCE metadata docs: https://cloud.google.com/compute/docs/storing-retrieving-metadata .. vi: textwidth=79 diff --git a/doc/rtd/topics/datasources/nocloud.rst b/doc/rtd/topics/datasources/nocloud.rst index 8ce656af0..6080d2881 100644 --- a/doc/rtd/topics/datasources/nocloud.rst +++ b/doc/rtd/topics/datasources/nocloud.rst @@ -149,7 +149,7 @@ be network configuration based on the filename. ethernets: interface0: match: - mac_address: "52:54:00:12:34:00" + macaddress: "52:54:00:12:34:00" set-name: interface0 addresses: - 192.168.1.10/255.255.255.0 diff --git a/doc/rtd/topics/datasources/vmware.rst b/doc/rtd/topics/datasources/vmware.rst index f1f48117d..de3de6af6 100644 --- a/doc/rtd/topics/datasources/vmware.rst +++ b/doc/rtd/topics/datasources/vmware.rst @@ -7,7 +7,7 @@ This datasource is for use with systems running on a VMware platform such as vSphere and currently supports the following data transports: -* `GuestInfo `_ keys +* `GuestInfo `_ keys Configuration ------------- @@ -263,7 +263,7 @@ this datasource: .. code-block:: bash - cloud-init clean + cloud-init clean --logs --machine-id Otherwise cloud-init may not run in first-boot mode. For more information on how the boot mode is determined, please see the @@ -311,7 +311,7 @@ this datasource: .. code-block:: shell - govc vm.power -vm "${VM}" -on + govc vm.power -on "${VM}" If all went according to plan, the CentOS box is: diff --git a/doc/rtd/topics/examples.rst b/doc/rtd/topics/examples.rst index 8ec8d8ab2..353e22d86 100644 --- a/doc/rtd/topics/examples.rst +++ b/doc/rtd/topics/examples.rst @@ -41,6 +41,13 @@ Install and run `chef`_ recipes :language: yaml :linenos: +Install and run `ansible`_ +========================== + +.. literalinclude:: ../../examples/cloud-config-ansible.txt + :language: yaml + :linenos: + Add primary apt repositories ============================ @@ -124,4 +131,4 @@ Create partitions and filesystems .. _chef: http://www.chef.io/chef/ .. _puppet: http://puppetlabs.com/ -.. vi: textwidth=79 +.. _ansible: https://docs.ansible.com/ansible/latest/ diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index 0f77fb156..2815f4928 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -151,6 +151,82 @@ provided to the system: As launching instances in the cloud can cost money and take a bit longer, sometimes it is easier to launch instances locally using Multipass or LXD: +Why did cloud-init never complete? +================================== + +To check if cloud-init is running still, run: + +.. code-block:: shell-session + + $ cloud-init status + +To wait for clous-init to complete, run: + +.. code-block:: shell-session + + $ cloud-init status --wait + +There are a number of reasons that cloud-init might never complete. This list +is not exhaustive, but attempts to enumerate potential causes: + +External reasons: +----------------- +- failed dependant services in the boot +- bugs in the kernel or drivers +- bugs in external userspace tools that are called by cloud-init + +Internal reasons: +----------------- +- a command in ``bootcmd`` or ``runcmd`` that never completes (ex: running + `cloud-init status --wait` will wait forever on itself and never complete) +- nonstandard configurations that disable timeouts or set extremely high + values ("never" is used in a loose sense here) + +How can I make a module run on every boot? +========================================== +Modules have a default frequency that can be overridden. This is done +by modifying the module list in ``/etc/cloud/cloud.cfg``. + +1. Change the module from a string (default) to a list. +2. Set the first list item to the module name and the second item to the + frequency. + +Example +------- +The following example demonstrates how to log boot times to a file every boot. + +Update ``/etc/cloud/cloud.cfg``: + +.. code-block:: yaml + :name: /etc/cloud/cloud.cfg + :emphasize-lines: 3 + + cloud_final_modules: + # list shortened for brevity + - [phone-home, always] + - final-message + - power-state-change + + + +Then your userdata could then be: + +.. code-block:: yaml + + #cloud-config + phone_home: + url: http://example.com/$INSTANCE_ID/ + post: all + + + +How can I test cloud-init locally before deploying to the cloud? +================================================================ + +Several different virtual machine and containerization tools can be used for +testing locally. Multipass, LXD, and qemu are described in this section. + + Multipass --------- @@ -212,17 +288,15 @@ launch this multiple times: The above examples all show how to pass user data. To pass other types of configuration data use the config option specified below: -+----------------+---------------------+ -| Data | Config Option | -+================+=====================+ -| user data | user.user-data | -+----------------+---------------------+ -| vendor data | user.vendor-data | -+----------------+---------------------+ -| metadata | user.meta-data | -+----------------+---------------------+ -| network config | user.network-config | -+----------------+---------------------+ ++----------------+---------------------------+ +| Data | Config Option | ++================+===========================+ +| user data | cloud-init.user-data | ++----------------+---------------------------+ +| vendor data | cloud-init.vendor-data | ++----------------+---------------------------+ +| network config | cloud-init.network-config | ++----------------+---------------------------+ See the LXD `Instance Configuration`_ docs for more info about configuration values or the LXD `Custom Network Configuration`_ document for more about @@ -232,8 +306,8 @@ custom network config. .. _Instance Configuration: https://linuxcontainers.org/lxd/docs/master/instances .. _Custom Network Configuration: https://linuxcontainers.org/lxd/docs/master/cloud-init -cloud-localds -------------- +QEMU +---- The `cloud-localds` command from the `cloud-utils`_ package generates a disk with user supplied data. The NoCloud datasouce allows users to provide their @@ -283,37 +357,56 @@ check out the :ref:`datasource_nocloud` page. .. _cloud-utils: https://github.com/canonical/cloud-utils/ Where can I learn more? -======================================== +======================= Below are some videos, blog posts, and white papers about cloud-init from a variety of sources. +Videos: + - `cloud-init - The Good Parts`_ -- `cloud-init Summit 2019`_ -- `Utilising cloud-init on Microsoft Azure (Whitepaper)`_ -- `Cloud Instance Initialization with cloud-init (Whitepaper)`_ -- `cloud-init Summit 2018`_ -- `cloud-init - The cross-cloud Magic Sauce (PDF)`_ -- `cloud-init Summit 2017`_ +- `Perfect Proxmox Template with Cloud Image and Cloud Init [proxmox, cloud-init, template]`_ - `cloud-init - Building clouds one Linux box at a time (Video)`_ -- `cloud-init - Building clouds one Linux box at a time (PDF)`_ - `Metadata and cloud-init`_ -- `The beauty of cloud-init`_ - `Introduction to cloud-init`_ +Blog Posts: + +- `cloud-init - The cross-cloud Magic Sauce (PDF)`_ +- `cloud-init - Building clouds one Linux box at a time (PDF)`_ +- `The beauty of cloud-init`_ +- `Cloud-init Getting Started [fedora, libvirt, cloud-init]`_ +- `Build Azure Devops Agents With Linux cloud-init for Dotnet Development [terraform, azure, devops, docker, dotnet, cloud-init]`_ +- `Cloud-init Getting Started [fedora, libvirt, cloud-init]`_ +- `Setup Neovim cloud-init Completion [neovim, yaml, Language Server Protocol, jsonschema, cloud-init]`_ + +Events: + +- `cloud-init Summit 2019`_ +- `cloud-init Summit 2018`_ +- `cloud-init Summit 2017`_ + + +Whitepapers: + +- `Utilising cloud-init on Microsoft Azure (Whitepaper)`_ +- `Cloud Instance Initialization with cloud-init (Whitepaper)`_ + .. _cloud-init - The Good Parts: https://www.youtube.com/watch?v=2_m6EUo6VOI -.. _cloud-init Summit 2019: https://powersj.io/post/cloud-init-summit19/ .. _Utilising cloud-init on Microsoft Azure (Whitepaper): https://ubuntu.com/engage/azure-cloud-init-whitepaper .. _Cloud Instance Initialization with cloud-init (Whitepaper): https://ubuntu.com/blog/cloud-instance-initialisation-with-cloud-init -.. _cloud-init Summit 2018: https://powersj.io/post/cloud-init-summit18/ + .. _cloud-init - The cross-cloud Magic Sauce (PDF): https://events.linuxfoundation.org/wp-content/uploads/2017/12/cloud-init-The-cross-cloud-Magic-Sauce-Scott-Moser-Chad-Smith-Canonical.pdf -.. _cloud-init Summit 2017: https://powersj.io/post/cloud-init-summit17/ .. _cloud-init - Building clouds one Linux box at a time (Video): https://www.youtube.com/watch?v=1joQfUZQcPg -.. _cloud-init - Building clouds one Linux box at a time (PDF): https://annex.debconf.org/debconf-share/debconf17/slides/164-cloud-init_Building_clouds_one_Linux_box_at_a_time.pdf +.. _cloud-init - Building clouds one Linux box at a time (PDF): https://web.archive.org/web/20181111020605/https://annex.debconf.org/debconf-share/debconf17/slides/164-cloud-init_Building_clouds_one_Linux_box_at_a_time.pdf .. _Metadata and cloud-init: https://www.youtube.com/watch?v=RHVhIWifVqU -.. _The beauty of cloud-init: http://brandon.fuller.name/archives/2011/05/02/06.40.57/ +.. _The beauty of cloud-init: https://web.archive.org/web/20180830161317/http://brandon.fuller.name/archives/2011/05/02/06.40.57/ .. _Introduction to cloud-init: http://www.youtube.com/watch?v=-zL3BdbKyGY -.. Blog Post: [terraform, azure, devops, docker, dotnet, cloud-init] https://codingsoul.org/2022/04/25/build-azure-devops-agents-with-linux-cloud-init-for-dotnet-development/ -.. Youtube: [proxmox, cloud-init, template] https://www.youtube.com/watch?v=shiIi38cJe4 +.. _Build Azure Devops Agents With Linux cloud-init for Dotnet Development [terraform, azure, devops, docker, dotnet, cloud-init]: https://codingsoul.org/2022/04/25/build-azure-devops-agents-with-linux-cloud-init-for-dotnet-development/ +.. _Perfect Proxmox Template with Cloud Image and Cloud Init [proxmox, cloud-init, template]: https://www.youtube.com/watch?v=shiIi38cJe4 +.. _Cloud-init Getting Started [fedora, libvirt, cloud-init]: https://blog.while-true-do.io/cloud-init-getting-started/ +.. _Setup Neovim cloud-init Completion [neovim, yaml, Language Server Protocol, jsonschema, cloud-init]: https://phoenix-labs.xyz/blog/setup-neovim-cloud-init-completion/ -.. vi: textwidth=79 +.. _cloud-init Summit 2019: https://powersj.io/post/cloud-init-summit19/ +.. _cloud-init Summit 2018: https://powersj.io/post/cloud-init-summit18/ +.. _cloud-init Summit 2017: https://powersj.io/post/cloud-init-summit17/ diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst index a4b772a24..7d75d168a 100644 --- a/doc/rtd/topics/format.rst +++ b/doc/rtd/topics/format.rst @@ -66,7 +66,7 @@ Kernel Command Line When using the :ref:`datasource_nocloud` datasource, users can pass user data via the kernel command line parameters. See the :ref:`datasource_nocloud` -datasource documentation for more details. +datasource and :ref:`kernel_cmdline` documentations for more details. Gzip Compressed Content ======================= diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 0c44d04e1..cf2f7e108 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -4,6 +4,12 @@ Instance Metadata ***************** +.. toctree:: + :maxdepth: 1 + :hidden: + + kernel-cmdline.rst + What is instance data? ======================== @@ -16,6 +22,7 @@ comes from any number of sources: * cloud-config seed files in the booted cloud image or distribution * vendordata provided from files or cloud metadata services * userdata provided at instance creation +* :ref:`kernel_cmdline` Each cloud provider presents unique configuration metadata in different formats to the instance. Cloud-init provides a cache of any crawled metadata diff --git a/doc/rtd/topics/kernel-cmdline.rst b/doc/rtd/topics/kernel-cmdline.rst new file mode 100644 index 000000000..4aa028556 --- /dev/null +++ b/doc/rtd/topics/kernel-cmdline.rst @@ -0,0 +1,71 @@ +.. _kernel_cmdline: + +******************* +Kernel Command Line +******************* + +In order to allow an ephemeral, or otherwise pristine image to +receive some configuration, cloud-init will read a url directed by +the kernel command line and proceed as if its data had previously existed. + +This allows for configuring a meta-data service, or some other data. + +.. note:: + + That usage of the kernel command line is somewhat of a last resort, + as it requires knowing in advance the correct command line or modifying + the boot loader to append data. + +For example, when ``cloud-init init --local`` runs, it will check to +see if ``cloud-config-url`` appears in key/value fashion +in the kernel command line as in: + +.. code-block:: text + + root=/dev/sda ro cloud-config-url=http://foo.bar.zee/abcde + +Cloud-init will then read the contents of the given url. +If the content starts with ``#cloud-config``, it will store +that data to the local filesystem in a static filename +``/etc/cloud/cloud.cfg.d/91_kernel_cmdline_url.cfg``, and consider it as +part of the config from that point forward. + +If that file exists already, it will not be overwritten, and the +`cloud-config-url` parameter is completely ignored. + +Then, when the DataSource runs, it will find that config already available. + +So, in order to be able to configure the MAAS DataSource by controlling the +kernel command line from outside the image, you can append: + + * ``cloud-config-url=http://your.url.here/abcdefg`` + +Then, have the following content at that url: + +.. code-block:: yaml + + #cloud-config + datasource: + MAAS: + metadata_url: http://mass-host.localdomain/source + consumer_key: Xh234sdkljf + token_key: kjfhgb3n + token_secret: 24uysdfx1w4 + +.. warning:: + + `url` kernel command line key is deprecated. + Please use `cloud-config-url` parameter instead" + +.. note:: + + Because ``cloud-config-url=`` is so very generic, in order to avoid false + positives, + cloud-init requires the content to start with ``#cloud-config`` in order + for it to be considered. + +.. note:: + + The ``cloud-config-url=`` is un-authed http GET, and contains credentials. + It could be set up to be randomly generated and also check source + address in order to be more secure. diff --git a/doc/rtd/topics/logging.rst b/doc/rtd/topics/logging.rst index 744e9bd42..f72b77c14 100644 --- a/doc/rtd/topics/logging.rst +++ b/doc/rtd/topics/logging.rst @@ -1,52 +1,15 @@ ******* Logging ******* -Cloud-init supports both local and remote logging configurable through python's -built-in logging configuration and through the cloud-init rsyslog module. +Cloud-init supports both local and remote logging configurable through +multiple configurations: -Command Output -============== -Cloud-init can redirect its stdout and stderr based on config given under the -``output`` config key. The output of any commands run by cloud-init and any -user or vendor scripts provided will also be included here. The ``output`` key -accepts a dictionary for configuration. Output files may be specified -individually for each stage (``init``, ``config``, and ``final``), or a single -key ``all`` may be used to specify output for all stages. - -The output for each stage may be specified as a dictionary of ``output`` and -``error`` keys, for stdout and stderr respectively, as a tuple with stdout -first and stderr second, or as a single string to use for both. The strings -passed to all of these keys are handled by the system shell, so any form of -redirection that can be used in bash is valid, including piping cloud-init's -output to ``tee``, or ``logger``. If only a filename is provided, cloud-init -will append its output to the file as though ``>>`` was specified. - -By default, cloud-init loads its output configuration from -``/etc/cloud/cloud.cfg.d/05_logging.cfg``. The default config directs both -stdout and stderr from all cloud-init stages to -``/var/log/cloud-init-output.log``. The default config is given as :: - - output: { all: "| tee -a /var/log/cloud-init-output.log" } - -For a more complex example, the following configuration would output the init -stage to ``/var/log/cloud-init.out`` and ``/var/log/cloud-init.err``, for -stdout and stderr respectively, replacing anything that was previously there. -For the config stage, it would pipe both stdout and stderr through ``tee -a -/var/log/cloud-config.log``. For the final stage it would append the output of -stdout and stderr to ``/var/log/cloud-final.out`` and -``/var/log/cloud-final.err`` respectively. :: - - output: - init: - output: "> /var/log/cloud-init.out" - error: "> /var/log/cloud-init.err" - config: "tee -a /var/log/cloud-config.log" - final: - - ">> /var/log/cloud-final.out" - - "/var/log/cloud-final.err" +- Python's built-in logging configuration +- Cloud-init's event reporting system +- The cloud-init rsyslog module Python Logging --------------- +============== Cloud-init uses the python logging module, and can accept config for this module using the standard python fileConfig format. Cloud-init looks for config for the logging module under the ``logcfg`` key. @@ -135,8 +98,131 @@ the default format string ``%(message)s``:: For additional information about configuring python's logging module, please see the documentation for `python logging config`_. -Rsyslog Module +Command Output -------------- +Cloud-init can redirect its stdout and stderr based on config given under the +``output`` config key. The output of any commands run by cloud-init and any +user or vendor scripts provided will also be included here. The ``output`` key +accepts a dictionary for configuration. Output files may be specified +individually for each stage (``init``, ``config``, and ``final``), or a single +key ``all`` may be used to specify output for all stages. + +The output for each stage may be specified as a dictionary of ``output`` and +``error`` keys, for stdout and stderr respectively, as a tuple with stdout +first and stderr second, or as a single string to use for both. The strings +passed to all of these keys are handled by the system shell, so any form of +redirection that can be used in bash is valid, including piping cloud-init's +output to ``tee``, or ``logger``. If only a filename is provided, cloud-init +will append its output to the file as though ``>>`` was specified. + +By default, cloud-init loads its output configuration from +``/etc/cloud/cloud.cfg.d/05_logging.cfg``. The default config directs both +stdout and stderr from all cloud-init stages to +``/var/log/cloud-init-output.log``. The default config is given as :: + + output: { all: "| tee -a /var/log/cloud-init-output.log" } + +For a more complex example, the following configuration would output the init +stage to ``/var/log/cloud-init.out`` and ``/var/log/cloud-init.err``, for +stdout and stderr respectively, replacing anything that was previously there. +For the config stage, it would pipe both stdout and stderr through ``tee -a +/var/log/cloud-config.log``. For the final stage it would append the output of +stdout and stderr to ``/var/log/cloud-final.out`` and +``/var/log/cloud-final.err`` respectively. :: + + output: + init: + output: "> /var/log/cloud-init.out" + error: "> /var/log/cloud-init.err" + config: "tee -a /var/log/cloud-config.log" + final: + - ">> /var/log/cloud-final.out" + - "/var/log/cloud-final.err" + +Event Reporting +=============== +Cloud-init contains an eventing system that allows events to emitted +to a variety of destinations. + +3 configurations are available for reporting events: + +- **webhook**: POST to a web server +- **log**: Write to the cloud-init log at configurable log level +- **stdout**: Print to stdout + +The default configuration is to emit events to the cloud-init log file +at ``DEBUG`` level. + +Event reporting can be configured using the ``reporting`` key in +cloud-config userdata. + +Configuration +------------- + +**webhook** + +.. code-block:: yaml + + reporting: + : + type: webhook + endpoint: + timeout: + retries: + consumer_key: + token_key: + token_secret: + consumer_secret: + +``endpoint`` is the only additional required key when specifying +``type: webhook``. + +**log** + +.. code-block:: yaml + + reporting: + : + type: log + level: + +``level`` is optional and defaults to "DEBUG". + +**print** + +.. code-block:: yaml + + reporting: + : + type: print + + +Example +^^^^^^^ + +The follow example shows configuration for all three sources: + +.. code-block:: yaml + + #cloud-config + reporting: + webserver: + type: webhook + endpoint: "http://10.0.0.1:55555/asdf" + timeout: 5 + retries: 3 + consumer_key: + token_key: + token_secret: + consumer_secret: + info_log: + type: log + level: WARN + stdout: + type: print + +Rsyslog Module +============== Cloud-init's ``cc_rsyslog`` module allows for fully customizable rsyslog configuration under the ``rsyslog`` config key. The simplest way to use the rsyslog module is by specifying remote servers under the ``remotes`` diff --git a/doc/rtd/topics/module_creation.rst b/doc/rtd/topics/module_creation.rst index b09cd2cc3..12cfdb00b 100644 --- a/doc/rtd/topics/module_creation.rst +++ b/doc/rtd/topics/module_creation.rst @@ -34,6 +34,7 @@ Example "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, + "activate_by_schema_keys": ["example_key, example_other_key"], "examples": [ "example_key: example_value", "example_other_key: ['value', 2]", @@ -82,6 +83,10 @@ Guidelines would be a significant change to the instance metadata. An example could be an instance being moved to a different subnet. + * ``activate_by_schema_keys``: (Optional) List of cloud-config keys that will + activate this module. When this list not empty, the config module will be + skipped unless one of the ``activate_by_schema_keys`` are present in merged + cloud-config instance-data. * ``examples``: Lists examples of any cloud-config keys this module reacts to. These examples will be rendered in the module reference documentation and will automatically be tested against the defined schema @@ -111,7 +116,7 @@ in the ``cloud_final_modules`` section before the ``final-message`` module. .. _MetaSchema: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/config/schema.py#L58 .. _OSFAMILIES: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/distros/__init__.py#L35 .. _settings.py: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/settings.py#L66 -.. _cloud-init-schema.json: https://github.com/canonical/cloud-init/blob/main/cloudinit/config/cloud-init-schema.json +.. _cloud-init-schema.json: https://github.com/canonical/cloud-init/blob/main/cloudinit/config/schemas/versions.schema.cloud-config.json .. _cloud.cfg.tmpl: https://github.com/canonical/cloud-init/blob/main/config/cloud.cfg.tmpl .. _cloud_init_modules: https://github.com/canonical/cloud-init/blob/b4746b6aed7660510071395e70b2d6233fbdc3ab/config/cloud.cfg.tmpl#L70 .. _cloud_config_modules: https://github.com/canonical/cloud-init/blob/b4746b6aed7660510071395e70b2d6233fbdc3ab/config/cloud.cfg.tmpl#L101 diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 4bfb27cf4..cbe0f5d79 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -5,6 +5,7 @@ Module Reference **************** .. contents:: Table of Contents +.. automodule:: cloudinit.config.cc_ansible .. automodule:: cloudinit.config.cc_apk_configure .. automodule:: cloudinit.config.cc_apt_configure .. automodule:: cloudinit.config.cc_apt_pipelining @@ -12,7 +13,6 @@ Module Reference .. automodule:: cloudinit.config.cc_byobu .. automodule:: cloudinit.config.cc_ca_certs .. automodule:: cloudinit.config.cc_chef -.. automodule:: cloudinit.config.cc_debug .. automodule:: cloudinit.config.cc_disable_ec2_metadata .. automodule:: cloudinit.config.cc_disk_setup .. automodule:: cloudinit.config.cc_fan @@ -55,10 +55,12 @@ Module Reference .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone .. automodule:: cloudinit.config.cc_ubuntu_advantage +.. automodule:: cloudinit.config.cc_ubuntu_autoinstall .. automodule:: cloudinit.config.cc_ubuntu_drivers .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname .. automodule:: cloudinit.config.cc_users_groups +.. automodule:: cloudinit.config.cc_wireguard .. automodule:: cloudinit.config.cc_write_files .. automodule:: cloudinit.config.cc_yum_add_repo .. automodule:: cloudinit.config.cc_zypper_add_repo diff --git a/doc/rtd/topics/network-config-format-v2.rst b/doc/rtd/topics/network-config-format-v2.rst index c1bf05d1b..3080c6d4c 100644 --- a/doc/rtd/topics/network-config-format-v2.rst +++ b/doc/rtd/topics/network-config-format-v2.rst @@ -338,7 +338,7 @@ Set whether to set all slaves to the same MAC address when adding them to the bond, or how else the system should handle MAC addresses. The possible values are ``none``, ``active``, and ``follow``. -**gratuitious-arp**: <*(scalar)>* +**gratuitous-arp**: <*(scalar)>* Specify how many ARP packets to send after failover. Once a link is up on a new slave, a notification is sent and possibly repeated if diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index c461a3fe4..3e48555f7 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -80,7 +80,8 @@ Disabling Network Activation Some datasources may not be initialized until after network has been brought up. In this case, cloud-init will attempt to bring up the interfaces specified -by the datasource metadata. +by the datasource metadata using a network activator discovered by +`cloudinit.net.activators.select_activators`_. This behavior can be disabled in the cloud-init configuration dictionary, merged from ``/etc/cloud/cloud.cfg`` and ``/etc/cloud/cloud.cfg.d/*``:: @@ -188,6 +189,15 @@ generated configuration into an internal network configuration state. From this state `Cloud-init`_ delegates rendering of the configuration to Distro supported formats. The following ``renderers`` are supported in cloud-init: +- **NetworkManager** + +`NetworkManager `_ is the standard Linux network +configuration tool suite. It supports a wide range of networking setups. +Configuration is typically stored in ``/etc/NetworkManager``. + +It is the default for a number of Linux distributions, notably Fedora; +CentOS/RHEL; and derivatives. + - **ENI** /etc/network/interfaces or ``ENI`` is supported by the ``ifupdown`` package @@ -206,6 +216,15 @@ network configuration for supported backends such as ``systemd-networkd`` and Sysconfig format is used by RHEL, CentOS, Fedora and other derivatives. +- **NetBSD, OpenBSD, FreeBSD** + +Network renders supporting BSD releases which typically write configuration to +``/etc/rc.conf``. Unique to BSD renderers is that each renderer also calls +something akin to `FreeBSD.start_services`_ which will invoke applicable +network services to setup the network, making network activators unneeded +for BSD flavors at the moment. + + Network Output Policy ===================== @@ -215,6 +234,19 @@ is as follows: - ENI - Sysconfig - Netplan +- NetworkManager +- FreeBSD +- NetBSD +- OpenBSD +- Networkd + +The default policy for selecting a network ``activator`` in order of preference +is as follows: +- ENI: using `ifup`, `ifdown` to manage device setup/teardown +- Netplan: using `netplan apply` to manage device setup/teardown +- NetworkManager: using `nmcli` to manage device setup/teardown +- Networkd: using `ip` to manage device setup/teardown + When applying the policy, `Cloud-init`_ checks if the current instance has the correct binaries and paths to support the renderer. The first renderer that @@ -223,7 +255,8 @@ supplying an updated configuration in cloud-config. :: system_info: network: - renderers: ['netplan', 'eni', 'sysconfig', 'freebsd', 'netbsd', 'openbsd'] + renderers: ['netplan', 'network-manager', 'eni', 'sysconfig', 'freebsd', 'netbsd', 'openbsd'] + activators: ['eni', 'netplan', 'network-manager', 'networkd'] Network Configuration Tools @@ -280,10 +313,12 @@ Example output converting V2 to sysconfig: .. _Cloud-init: https://launchpad.net/cloud-init -.. _DigitalOcean JSON metadata: https://developers.digitalocean.com/documentation/metadata/#network-interfaces-index +.. _DigitalOcean JSON metadata: https://developers.digitalocean.com/documentation/metadata/ .. _OpenStack Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html .. _SmartOS JSON Metadata: https://eng.joyent.com/mdata/datadict.html .. _UpCloud JSON metadata: https://developers.upcloud.com/1.3/8-servers/#metadata-service .. _Vultr JSON metadata: https://www.vultr.com/metadata/ +.. _cloudinit.net.activators.select_activators: https://github.com/canonical/cloud-init/blob/main/cloudinit/net/activators.py#L279 +.. _FreeBSD.start_services: https://github.com/canonical/cloud-init/blob/main/cloudinit/net/freebsd.py#L28 .. vi: textwidth=79 diff --git a/doc/sources/kernel-cmdline.txt b/doc/sources/kernel-cmdline.txt deleted file mode 100644 index 4cbfd217d..000000000 --- a/doc/sources/kernel-cmdline.txt +++ /dev/null @@ -1,48 +0,0 @@ -In order to allow an ephemeral, or otherwise pristine image to -receive some configuration, cloud-init will read a url directed by -the kernel command line and proceed as if its data had previously existed. - -This allows for configuring a meta-data service, or some other data. - -Note, that usage of the kernel command line is somewhat of a last resort, -as it requires knowing in advance the correct command line or modifying -the boot loader to append data. - -For example, when 'cloud-init start' runs, it will check to -see if one of 'cloud-config-url' or 'url' appear in key/value fashion -in the kernel command line as in: - root=/dev/sda ro url=http://foo.bar.zee/abcde - -Cloud-init will then read the contents of the given url. -If the content starts with '#cloud-config', it will store -that data to the local filesystem in a static filename -'/etc/cloud/cloud.cfg.d/91_kernel_cmdline_url.cfg', and consider it as -part of the config from that point forward. - -If that file exists already, it will not be overwritten, and the url parameters -completely ignored. - -Then, when the DataSource runs, it will find that config already available. - -So, in able to configure the MAAS DataSource by controlling the kernel -command line from outside the image, you can append: - url=http://your.url.here/abcdefg -or - cloud-config-url=http://your.url.here/abcdefg - -Then, have the following content at that url: - #cloud-config - datasource: - MAAS: - metadata_url: http://mass-host.localdomain/source - consumer_key: Xh234sdkljf - token_key: kjfhgb3n - token_secret: 24uysdfx1w4 - -Notes: - * Because 'url=' is so very generic, in order to avoid false positives, - cloud-init requires the content to start with '#cloud-config' in order - for it to be considered. - * The url= is un-authed http GET, and contains credentials - It could be set up to be randomly generated and also check source - address in order to be more secure diff --git a/integration-requirements.txt b/integration-requirements.txt index 102553cb2..955030127 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,5 +1,5 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # -pycloudlib @ git+https://github.com/canonical/pycloudlib.git@675dffdc14224a03f8f0ba7212ecb3ca2a8a7083 +pycloudlib @ git+https://github.com/canonical/pycloudlib.git@7f5bf6e67cf79f31770c456196b2ce695c6ce165 pytest diff --git a/packages/bddeb b/packages/bddeb index b009021a0..fdb541d4a 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -34,7 +34,13 @@ DEBUILD_ARGS = ["-S", "-d"] def get_release_suffix(release): - """Given ubuntu release (xenial), return a suffix for package (~16.04.1)""" + """Given ubuntu release, return a suffix for package + + Examples: + --------- + >>> get_release_suffix("jammy") + '~22.04.1' + """ csv_path = "/usr/share/distro-info/ubuntu.csv" rels = {} # fields are version, codename, series, created, release, eol, eol-server @@ -150,10 +156,6 @@ def get_parser(): default=False, action='store_true') - parser.add_argument("--python2", dest="python2", - help=("build debs for python2 rather than python3"), - default=False, action='store_true') - parser.add_argument("--init-system", dest="init_system", help=("build deb with INIT_SYSTEM=xxx" " (default: %(default)s"), diff --git a/packages/debian/cloud-init.postrm b/packages/debian/cloud-init.postrm new file mode 100644 index 000000000..6cb9f54ee --- /dev/null +++ b/packages/debian/cloud-init.postrm @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +cleanup_sshd_config() { + rm -f "/etc/ssh/sshd_config.d/50-cloud-init.conf" +} + +if [ "$1" = "purge" ]; then + cleanup_sshd_config +fi diff --git a/packages/debian/control.in b/packages/debian/control.in index 5bb915a91..30cf406b7 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -12,7 +12,8 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, iproute2, - isc-dhcp-client + isc-dhcp-client, + python3-debconf Recommends: eatmydata, sudo, software-properties-common, gdisk Suggests: ssh-import-id, openssh-server Description: Init scripts for cloud instances diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index 36e6b38f7..8ba27e855 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -3,7 +3,8 @@ "build-requires" : [ "debhelper", "dh-python", - "dh-systemd" + "dh-systemd", + "python3-debconf" ], "renames" : { "pyyaml" : "python3-yaml", diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 1491822bb..5cbf828ab 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -48,11 +48,6 @@ BuildRequires: {{r}} Requires: dmidecode %endif -# python2.6 needs argparse -%if "%{?el6}" == "1" -Requires: python-argparse -%endif - # Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json. # Install them as BuildRequires too as they're used for testing. @@ -171,7 +166,7 @@ fi %files -/lib/udev/rules.d/66-azure-ephemeral.rules +%{_udevrulesdir}/66-azure-ephemeral.rules %if "%{init_system}" == "systemd" /usr/lib/systemd/system-generators/cloud-init-generator @@ -197,6 +192,8 @@ fi # Configs %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg +%dir %{_sysconfdir}/cloud/clean.d +%config(noreplace) %{_sysconfdir}/cloud/clean.d/README %dir %{_sysconfdir}/cloud/cloud.cfg.d %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/*.cfg %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/README diff --git a/packages/suse/cloud-init.spec.in b/packages/suse/cloud-init.spec.in index da8107b40..2586f2487 100644 --- a/packages/suse/cloud-init.spec.in +++ b/packages/suse/cloud-init.spec.in @@ -114,6 +114,8 @@ version_pys=$(cd "%{buildroot}" && find . -name version.py -type f) %doc %{_defaultdocdir}/cloud-init/* # Configs +%dir %{_sysconfdir}/cloud/clean.d +%config(noreplace) %{_sysconfdir}/cloud/clean.d/README %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg %dir %{_sysconfdir}/cloud/cloud.cfg.d %config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/*.cfg diff --git a/pyproject.toml b/pyproject.toml index 1aac03a85..2ee261212 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,92 +9,25 @@ skip = ["cloudinit/cmd/main.py", ".tox", "packages", "tools"] [tool.mypy] follow_imports = "silent" -exclude=[ - '^cloudinit/apport\.py$', - '^cloudinit/cmd/query\.py$', - '^cloudinit/config/cc_chef\.py$', - '^cloudinit/config/cc_keyboard\.py$', - '^cloudinit/config/cc_landscape\.py$', - '^cloudinit/config/cc_mcollective\.py$', - '^cloudinit/config/cc_rsyslog\.py$', - '^cloudinit/config/cc_write_files_deferred\.py$', - '^cloudinit/config/cc_zypper_add_repo\.py$', - '^cloudinit/config/schema\.py$', - '^cloudinit/distros/bsd\.py$', - '^cloudinit/distros/freebsd\.py$', - '^cloudinit/distros/parsers/networkmanager_conf\.py$', - '^cloudinit/distros/parsers/resolv_conf\.py$', - '^cloudinit/distros/parsers/sys_conf\.py$', - '^cloudinit/dmi\.py$', - '^cloudinit/features\.py$', - '^cloudinit/handlers/cloud_config\.py$', - '^cloudinit/handlers/jinja_template\.py$', - '^cloudinit/net/__init__\.py$', - '^cloudinit/net/dhcp\.py$', - '^cloudinit/net/netplan\.py$', - '^cloudinit/net/sysconfig\.py$', - '^cloudinit/serial\.py$', - '^cloudinit/sources/DataSourceAliYun\.py$', - '^cloudinit/sources/DataSourceLXD\.py$', - '^cloudinit/sources/DataSourceOracle\.py$', - '^cloudinit/sources/DataSourceScaleway\.py$', - '^cloudinit/sources/DataSourceSmartOS\.py$', - '^cloudinit/sources/DataSourceVMware\.py$', - '^cloudinit/sources/__init__\.py$', - '^cloudinit/sources/helpers/vmware/imc/config_file\.py$', - '^cloudinit/templater\.py$', - '^cloudinit/url_helper\.py$', - '^conftest\.py$', - '^doc/rtd/conf\.py$', - '^setup\.py$', - '^tests/integration_tests/clouds\.py$', - '^tests/integration_tests/conftest\.py$', - '^tests/integration_tests/instances\.py$', - '^tests/integration_tests/integration_settings\.py$', - '^tests/integration_tests/modules/test_disk_setup\.py$', - '^tests/integration_tests/modules/test_growpart\.py$', - '^tests/integration_tests/modules/test_ssh_keysfile\.py$', - '^tests/unittests/__init__\.py$', - '^tests/unittests/cmd/test_clean\.py$', - '^tests/unittests/cmd/test_cloud_id\.py$', - '^tests/unittests/cmd/test_main\.py$', - '^tests/unittests/config/test_cc_chef\.py$', - '^tests/unittests/config/test_cc_landscape\.py$', - '^tests/unittests/config/test_cc_locale\.py$', - '^tests/unittests/config/test_cc_mcollective\.py$', - '^tests/unittests/config/test_cc_rh_subscription\.py$', - '^tests/unittests/config/test_cc_set_hostname\.py$', - '^tests/unittests/config/test_cc_snap\.py$', - '^tests/unittests/config/test_cc_timezone\.py$', - '^tests/unittests/config/test_cc_ubuntu_advantage\.py$', - '^tests/unittests/config/test_cc_ubuntu_drivers\.py$', - '^tests/unittests/config/test_schema\.py$', - '^tests/unittests/helpers\.py$', - '^tests/unittests/net/test_dhcp\.py$', - '^tests/unittests/net/test_init\.py$', - '^tests/unittests/sources/test_aliyun\.py$', - '^tests/unittests/sources/test_ec2\.py$', - '^tests/unittests/sources/test_exoscale\.py$', - '^tests/unittests/sources/test_gce\.py$', - '^tests/unittests/sources/test_lxd\.py$', - '^tests/unittests/sources/test_opennebula\.py$', - '^tests/unittests/sources/test_openstack\.py$', - '^tests/unittests/sources/test_rbx\.py$', - '^tests/unittests/sources/test_scaleway\.py$', - '^tests/unittests/sources/test_smartos\.py$', - '^tests/unittests/test_data\.py$', - '^tests/unittests/test_ds_identify\.py$', - '^tests/unittests/test_ec2_util\.py$', - '^tests/unittests/test_net\.py$', - '^tests/unittests/test_net_activators\.py$', - '^tests/unittests/test_persistence\.py$', - '^tests/unittests/test_sshutil\.py$', - '^tests/unittests/test_subp\.py$', - '^tests/unittests/test_templating\.py$', - '^tests/unittests/test_url_helper\.py$', - '^tools/mock-meta\.py$', -] +warn_unused_ignores = "true" +warn_redundant_casts = "true" +exclude=[] [[tool.mypy.overrides]] -module = [ "httpretty", "pycloudlib.*" ] +module = [ + "apport.*", + "BaseHTTPServer", + "configobj", + "cloudinit.feature_overrides", + "debconf", + "httpretty", + "httplib", + "jsonpatch", + "netifaces", + "paramiko.*", + "pycloudlib.*", + "responses", + "serial", + "tests.integration_tests.user_settings" +] ignore_missing_imports = true diff --git a/requirements.txt b/requirements.txt index c4adc455b..edec46a7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,7 @@ oauthlib # This one is currently used only by the CloudSigma and SmartOS datasources. # If these datasources are removed, this is no longer needed. # -# This will not work in py2.6 so it is only optionally installed on -# python 2.7 and later. -# -# pyserial +pyserial # This is only needed for places where we need to support configs in a manner # that the built-in config parser is not sufficent (ie diff --git a/setup.py b/setup.py index 7ba0ee8c3..470dd7742 100644 --- a/setup.py +++ b/setup.py @@ -91,9 +91,8 @@ def render_tmpl(template, mode=None): in that file if user had something there. b.) debuild will complain that files are different outside of the debian directory.""" - # older versions of tox use bdist (xenial), and then install from there. # newer versions just use install. - if not (sys.argv[1] == "install" or sys.argv[1].startswith("bdist*")): + if not (sys.argv[1] == "install"): return template tmpl_ext = ".tmpl" @@ -277,6 +276,7 @@ def finalize_options(self): data_files = [ (ETC + "/cloud", [render_tmpl("config/cloud.cfg.tmpl")]), + (ETC + "/cloud/clean.d", glob("config/clean.d/*")), (ETC + "/cloud/cloud.cfg.d", glob("config/cloud.cfg.d/*")), (ETC + "/cloud/templates", glob("templates/*")), ( @@ -303,6 +303,11 @@ def finalize_options(self): ), ] if not platform.system().endswith("BSD"): + + RULES_PATH = LIB + if os.path.isfile("/etc/redhat-release"): + RULES_PATH = "/usr/lib" + data_files.extend( [ ( @@ -310,7 +315,7 @@ def finalize_options(self): ["tools/hook-network-manager"], ), (ETC + "/dhcp/dhclient-exit-hooks.d/", ["tools/hook-dhclient"]), - (LIB + "/udev/rules.d", [f for f in glob("udev/*.rules")]), + (RULES_PATH + "/udev/rules.d", [f for f in glob("udev/*.rules")]), ( ETC + "/systemd/system/sshd-keygen@.service.d/", ["systemd/disable-sshd-keygen-if-cloud-init-active.conf"], diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index 6403041e5..7666367ca 100644 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -21,7 +21,7 @@ CLOUD_SYSTEM_TARGET="/usr/lib/systemd/system/cloud-init.target" CLOUD_SYSTEM_TARGET="/lib/systemd/system/cloud-init.target" {% endif %} {% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", - "miraclelinux", "openEuler", "rhel", "rocky", "virtuozzo"] %} + "miraclelinux", "openEuler", "openmandriva", "rhel", "rocky", "virtuozzo"] %} dsidentify="/usr/libexec/cloud-init/ds-identify" {% else %} dsidentify="/usr/lib/cloud-init/ds-identify" diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index cf0a0205a..aec505990 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -13,7 +13,7 @@ After=nss-lookup.target After=networking.service {% endif %} {% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", - "miraclelinux", "openEuler", "rhel", "rocky", "virtuozzo"] %} + "miraclelinux", "openEuler", "openmandriva", "rhel", "rocky", "virtuozzo"] %} After=network.service After=NetworkManager.service {% endif %} diff --git a/templates/chrony.conf.centos.tmpl b/templates/chrony.conf.centos.tmpl new file mode 100644 index 000000000..5b3542ef7 --- /dev/null +++ b/templates/chrony.conf.centos.tmpl @@ -0,0 +1,45 @@ +## template:jinja +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +# Allow the system clock to be stepped in the first three updates +# if its offset is larger than 1 second. +makestep 1.0 3 + +# Enable kernel synchronization of the real-time clock (RTC). +rtcsync + +# Enable hardware timestamping on all interfaces that support it. +#hwtimestamp * + +# Increase the minimum number of selectable sources required to adjust +# the system clock. +#minsources 2 + +# Allow NTP client access from local network. +#allow 192.168.0.0/16 + +# Serve time even if not synchronized to a time source. +#local stratum 10 + +# Specify file containing keys for NTP authentication. +#keyfile /etc/chrony.keys + +# Specify directory for log files. +logdir /var/log/chrony + +# Select which information is logged. +#log measurements statistics tracking diff --git a/tests/data/old_pickles/focal-azure-20.1-10-g71af48df-0ubuntu5.pkl b/tests/data/old_pickles/focal-azure-20.1-10-g71af48df-0ubuntu5.pkl new file mode 100644 index 000000000..7f6000c77 Binary files /dev/null and b/tests/data/old_pickles/focal-azure-20.1-10-g71af48df-0ubuntu5.pkl differ diff --git a/tests/hypothesis.py b/tests/hypothesis.py new file mode 100644 index 000000000..def9de294 --- /dev/null +++ b/tests/hypothesis.py @@ -0,0 +1,20 @@ +try: + from hypothesis import given + + HAS_HYPOTHESIS = True +except ImportError: + HAS_HYPOTHESIS = False + + from unittest import mock + + def given(*_, **__): # type: ignore + """Dummy implementation to make pytest collection pass""" + + @mock.Mock # Add mock to fulfill the expected hypothesis value + def run_test(item): + return item + + return run_test + + +__all__ = ["given", "HAS_HYPOTHESIS"] diff --git a/tests/hypothesis_jsonschema.py b/tests/hypothesis_jsonschema.py new file mode 100644 index 000000000..cce7a9dac --- /dev/null +++ b/tests/hypothesis_jsonschema.py @@ -0,0 +1,12 @@ +try: + from hypothesis_jsonschema import from_schema + + HAS_HYPOTHESIS_JSONSCHEMA = True +except ImportError: + HAS_HYPOTHESIS_JSONSCHEMA = False + + def from_schema(*_, **__): # type: ignore + pass + + +__all__ = ["from_schema", "HAS_HYPOTHESIS_JSONSCHEMA"] diff --git a/tests/integration_tests/bugs/test_lp1835584.py b/tests/integration_tests/bugs/test_lp1835584.py index 765d73ef9..4e732446e 100644 --- a/tests/integration_tests/bugs/test_lp1835584.py +++ b/tests/integration_tests/bugs/test_lp1835584.py @@ -12,8 +12,6 @@ recreate ssh hostkeys across reboot (due to detecting an instance_id change). This currently only affects linux-azure-fips -> linux-azure on Bionic. -This test won't run on Xenial because both linux-azure-fips and linux-azure -report uppercase product_uuids. The test will launch a specific Bionic Ubuntu PRO FIPS image which has a linux-azure-fips kernel known to report product_uuid as uppercase. Then upgrade @@ -30,15 +28,12 @@ import re import pytest +from pycloudlib.cloud import ImageType from tests.integration_tests.clouds import ImageSpecification, IntegrationCloud from tests.integration_tests.conftest import get_validated_source from tests.integration_tests.instances import IntegrationInstance -IMG_AZURE_UBUNTU_PRO_FIPS_BIONIC = ( - "Canonical:0001-com-ubuntu-pro-bionic-fips:pro-fips-18_04:18.04.202010201" -) - def _check_iid_insensitive_across_kernel_upgrade( instance: IntegrationInstance, @@ -73,6 +68,7 @@ def _check_iid_insensitive_across_kernel_upgrade( @pytest.mark.azure +@pytest.mark.integration_cloud_args(image_type=ImageType.PRO_FIPS) def test_azure_kernel_upgrade_case_insensitive_uuid( session_cloud: IntegrationCloud, ): @@ -88,9 +84,12 @@ def test_azure_kernel_upgrade_case_insensitive_uuid( pytest.skip( "Provide CLOUD_INIT_SOURCE to install expected working cloud-init" ) - image_id = IMG_AZURE_UBUNTU_PRO_FIPS_BIONIC with session_cloud.launch( - launch_kwargs={"image_id": image_id} + launch_kwargs={ + "image_id": session_cloud.cloud_instance.daily_image( + cfg_image_spec.image_id, image_type=ImageType.PRO_FIPS + ) + } ) as instance: # We can't use setup_image fixture here because we want to avoid # taking a snapshot or cleaning the booted machine after cloud-init diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 0e2e1deb2..6b959adea 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -18,9 +18,9 @@ LXDVirtualMachine, Openstack, ) -from pycloudlib.cloud import BaseCloud +from pycloudlib.cloud import BaseCloud, ImageType from pycloudlib.lxd.cloud import _BaseLXD -from pycloudlib.lxd.instance import LXDInstance +from pycloudlib.lxd.instance import BaseInstance, LXDInstance import cloudinit from cloudinit.subp import ProcessExecutionError, subp @@ -94,7 +94,12 @@ class IntegrationCloud(ABC): datasource: str cloud_instance: BaseCloud - def __init__(self, settings=integration_settings): + def __init__( + self, + image_type: ImageType = ImageType.GENERIC, + settings=integration_settings, + ): + self._image_type = image_type self.settings = settings self.cloud_instance: BaseCloud = self._get_cloud_instance() self.initial_image_id = self._get_initial_image() @@ -119,14 +124,15 @@ def emit_settings_to_log(self) -> None: def _get_cloud_instance(self): raise NotImplementedError - def _get_initial_image(self): + def _get_initial_image(self, **kwargs) -> str: image = ImageSpecification.from_os_image() try: - return self.cloud_instance.daily_image(image.image_id) - except (ValueError, IndexError): + return self.cloud_instance.daily_image(image.image_id, **kwargs) + except (ValueError, IndexError) as ex: + log.debug("Exception while executing `daily_image`: %s", ex) return image.image_id - def _perform_launch(self, launch_kwargs, **kwargs): + def _perform_launch(self, launch_kwargs, **kwargs) -> BaseInstance: pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs) return pycloudlib_instance @@ -145,10 +151,11 @@ def launch( "Instance id: %s", self.settings.EXISTING_INSTANCE_ID, ) - self.instance = self.cloud_instance.get_instance( + pycloudlib_instance = self.cloud_instance.get_instance( self.settings.EXISTING_INSTANCE_ID ) - return self.instance + instance = self.get_instance(pycloudlib_instance, settings) + return instance default_launch_kwargs = { "image_id": self.image_id, "user_data": user_data, @@ -174,7 +181,9 @@ def launch( log.info("image serial: %s", serial.split()[1]) return instance - def get_instance(self, cloud_instance, settings=integration_settings): + def get_instance( + self, cloud_instance, settings=integration_settings + ) -> IntegrationInstance: return IntegrationInstance(self, cloud_instance, settings) def destroy(self): @@ -205,6 +214,11 @@ class Ec2Cloud(IntegrationCloud): def _get_cloud_instance(self): return EC2(tag="ec2-integration-test") + def _get_initial_image(self, **kwargs) -> str: + return super()._get_initial_image( + image_type=self._image_type, **kwargs + ) + def _perform_launch(self, launch_kwargs, **kwargs): """Use a dual-stack VPC for cloud-init integration testing.""" if "vpc" not in launch_kwargs: @@ -231,6 +245,11 @@ def _get_cloud_instance(self): tag="gce-integration-test", ) + def _get_initial_image(self, **kwargs) -> str: + return super()._get_initial_image( + image_type=self._image_type, **kwargs + ) + class AzureCloud(IntegrationCloud): datasource = "azure" @@ -239,6 +258,11 @@ class AzureCloud(IntegrationCloud): def _get_cloud_instance(self): return Azure(tag="azure-integration-test") + def _get_initial_image(self, **kwargs) -> str: + return super()._get_initial_image( + image_type=self._image_type, **kwargs + ) + def destroy(self): if self.settings.KEEP_INSTANCE: log.info( @@ -363,7 +387,7 @@ def _get_cloud_instance(self): tag="openstack-integration-test", ) - def _get_initial_image(self): + def _get_initial_image(self, **kwargs): image = ImageSpecification.from_os_image() try: UUID(image.image_id) diff --git a/tests/integration_tests/cmd/test_schema.py b/tests/integration_tests/cmd/test_schema.py new file mode 100644 index 000000000..0d92f146b --- /dev/null +++ b/tests/integration_tests/cmd/test_schema.py @@ -0,0 +1,66 @@ +"""Tests for `cloud-init status`""" +from textwrap import dedent + +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + +USER_DATA = """\ +#cloud-config +apt_update: false +apt_upgrade: false +apt_reboot_if_required: false +""" + + +@pytest.mark.user_data(USER_DATA) +class TestSchemaDeprecations: + def test_clean_log(self, class_client: IntegrationInstance): + log = class_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log, ignore_deprecations=True) + assert "WARNING]: Deprecated cloud-config provided:" in log + assert "apt_reboot_if_required: DEPRECATED" in log + assert "apt_update: DEPRECATED" in log + assert "apt_upgrade: DEPRECATED" in log + + def test_schema_deprecations(self, class_client: IntegrationInstance): + """Test schema behavior with deprecated configs.""" + user_data_fn = "/root/user-data" + class_client.write_to_file(user_data_fn, USER_DATA) + + result = class_client.execute( + f"cloud-init schema --config-file {user_data_fn}" + ) + assert ( + result.ok + ), "`schema` cmd must return 0 even with deprecated configs" + assert not result.stderr + assert "Cloud config schema deprecations:" in result.stdout + assert "apt_update: DEPRECATED" in result.stdout + assert "apt_upgrade: DEPRECATED" in result.stdout + assert "apt_reboot_if_required: DEPRECATED" in result.stdout + + annotated_result = class_client.execute( + f"cloud-init schema --annotate --config-file {user_data_fn}" + ) + assert ( + annotated_result.ok + ), "`schema` cmd must return 0 even with deprecated configs" + assert not annotated_result.stderr + expected_output = dedent( + """\ + #cloud-config + apt_update: false\t\t# D1 + apt_upgrade: false\t\t# D2 + apt_reboot_if_required: false\t\t# D3 + + # Deprecations: ------------- + # D1: DEPRECATED: Dropped after April 2027. Use ``package_update``. Default: ``false`` + # D2: DEPRECATED: Dropped after April 2027. Use ``package_upgrade``. Default: ``false`` + # D3: DEPRECATED: Dropped after April 2027. Use ``package_reboot_if_required``. Default: ``false`` + + + Valid cloud-config: /root/user-data""" # noqa: E501 + ) + assert expected_output in annotated_result.stdout diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index ced883fdd..2582855de 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -32,7 +32,9 @@ def _remove_nocloud_dir_and_reboot(client: IntegrationInstance): # On Impish and below, NoCloud will be detected on an LXD container. # If we remove this directory, it will no longer be detected. client.execute("rm -rf /var/lib/cloud/seed/nocloud-net") + old_boot_id = client.instance.get_boot_id() client.execute("cloud-init clean --logs --reboot") + client.instance._wait_for_execute(old_boot_id=old_boot_id) @pytest.mark.ubuntu @@ -53,17 +55,15 @@ def test_wait_when_no_datasource(session_cloud: IntegrationCloud, setup_image): } ) as client: # We know this will be an LXD instance due to our pytest mark - client.instance.execute_via_ssh = False # type: ignore + client.instance.execute_via_ssh = False # pyright: ignore # No ubuntu user if cloud-init didn't run client.instance.username = "root" # Jammy and above will use LXD datasource by default if ImageSpecification.from_os_image().release in [ "bionic", "focal", - "impish", ]: _remove_nocloud_dir_and_reboot(client) status_out = _wait_for_cloud_init(client).stdout.strip() assert "status: disabled" in status_out - assert "Cloud-init disabled by cloud-init-generator" in status_out assert client.execute("cloud-init status --wait").ok diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index a90a5d49b..6157bad85 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -7,7 +7,7 @@ from contextlib import contextmanager from pathlib import Path from tarfile import TarFile -from typing import Dict, Type +from typing import Dict, Generator, Iterator, Type import pytest from pycloudlib.lxd.instance import LXDInstance @@ -93,20 +93,16 @@ def disable_subp_usage(request): @pytest.fixture(scope="session") -def session_cloud(): +def session_cloud() -> Generator[IntegrationCloud, None, None]: if integration_settings.PLATFORM not in platforms.keys(): raise ValueError( - "{} is an invalid PLATFORM specified in settings. " - "Must be one of {}".format( - integration_settings.PLATFORM, list(platforms.keys()) - ) + f"{integration_settings.PLATFORM} is an invalid PLATFORM " + f"specified in settings. Must be one of {list(platforms.keys())}" ) cloud = platforms[integration_settings.PLATFORM]() cloud.emit_settings_to_log() - yield cloud - cloud.destroy() @@ -130,9 +126,7 @@ def get_validated_source( return CloudInitSource.DEB_PACKAGE elif source == "UPGRADE": return CloudInitSource.UPGRADE - raise ValueError( - "Invalid value for CLOUD_INIT_SOURCE setting: {}".format(source) - ) + raise ValueError(f"Invalid value for CLOUD_INIT_SOURCE setting: {source}") @pytest.fixture(scope="session") @@ -141,7 +135,6 @@ def setup_image(session_cloud: IntegrationCloud, request): So we can launch instances / run tests with the correct image """ - source = get_validated_source(session_cloud) if not source.installs_new_version(): return @@ -218,7 +211,9 @@ def _collect_logs( @contextmanager -def _client(request, fixture_utils, session_cloud: IntegrationCloud): +def _client( + request, fixture_utils, session_cloud: IntegrationCloud +) -> Iterator[IntegrationInstance]: """Fixture implementation for the client fixtures. Launch the dynamic IntegrationClient instance using any provided @@ -268,21 +263,27 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud): @pytest.fixture -def client(request, fixture_utils, session_cloud, setup_image): +def client( + request, fixture_utils, session_cloud, setup_image +) -> Iterator[IntegrationInstance]: """Provide a client that runs for every test.""" with _client(request, fixture_utils, session_cloud) as client: yield client @pytest.fixture(scope="module") -def module_client(request, fixture_utils, session_cloud, setup_image): +def module_client( + request, fixture_utils, session_cloud, setup_image +) -> Iterator[IntegrationInstance]: """Provide a client that runs once per module.""" with _client(request, fixture_utils, session_cloud) as client: yield client @pytest.fixture(scope="class") -def class_client(request, fixture_utils, session_cloud, setup_image): +def class_client( + request, fixture_utils, session_cloud, setup_image +) -> Iterator[IntegrationInstance]: """Provide a client that runs once per class.""" with _client(request, fixture_utils, session_cloud) as client: yield client diff --git a/tests/integration_tests/datasources/test_ec2_ipv6.py b/tests/integration_tests/datasources/test_ec2_ipv6.py index 8cde4dc9e..7bb45b402 100644 --- a/tests/integration_tests/datasources/test_ec2_ipv6.py +++ b/tests/integration_tests/datasources/test_ec2_ipv6.py @@ -10,9 +10,7 @@ def _test_crawl(client, ip): assert client.execute("cloud-init init --local").ok log = client.read_from_file("/var/log/cloud-init.log") assert f"Using metadata source: '{ip}'" in log - result = re.findall( - r"Crawl of metadata service took (\d+.\d+) seconds", log - ) + result = re.findall(r"Crawl of metadata service.* (\d+.\d+) seconds", log) if len(result) != 1: pytest.fail(f"Expected 1 metadata crawl time, got {result}") # 20 would still be a crazy long time for metadata service to crawl, @@ -41,3 +39,11 @@ def test_dual_stack(client: IntegrationInstance): # Block IPv6 requests assert client.execute("ip6tables -I OUTPUT -d fd00:ec2::254 -j REJECT").ok _test_crawl(client, "http://169.254.169.254") + + # Force NoDHCPLeaseError (by removing dhclient) and assert ipv6 still works + # Destructive test goes last + # dhclient is at /sbin/dhclient on bionic but /usr/sbin/dhclient elseware + assert client.execute("rm $(which dhclient)").ok + client.restart() + log = client.read_from_file("/var/log/cloud-init.log") + assert "Crawl of metadata service using link-local ipv6 took" in log diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py index f72b1b4b4..899ea9351 100644 --- a/tests/integration_tests/datasources/test_lxd_discovery.py +++ b/tests/integration_tests/datasources/test_lxd_discovery.py @@ -8,7 +8,7 @@ from tests.integration_tests.util import verify_clean_log -def _customize_envionment(client: IntegrationInstance): +def _customize_environment(client: IntegrationInstance): # Assert our platform can detect LXD during systemd generator timeframe. ds_id_log = client.execute("cat /run/cloud-init/ds-identify.log").stdout assert "check for 'LXD' returned found" in ds_id_log @@ -54,7 +54,7 @@ def _customize_envionment(client: IntegrationInstance): def test_lxd_datasource_discovery(client: IntegrationInstance): """Test that DataSourceLXD is detected instead of NoCloud.""" - _customize_envionment(client) + _customize_environment(client) result = client.execute("cloud-init status --wait --long") if not result.ok: raise AssertionError("cloud-init failed:\n%s", result.stderr) @@ -113,7 +113,6 @@ def test_lxd_datasource_discovery(client: IntegrationInstance): if ImageSpecification.from_os_image().release in [ "bionic", "focal", - "impish", ]: # Assert NoCloud seed files are still present in non-Jammy images # and that NoCloud seed files provide the same content as LXD socket. diff --git a/tests/integration_tests/datasources/test_network_dependency.py b/tests/integration_tests/datasources/test_network_dependency.py index 32ac70536..bd7fe658f 100644 --- a/tests/integration_tests/datasources/test_network_dependency.py +++ b/tests/integration_tests/datasources/test_network_dependency.py @@ -3,7 +3,7 @@ from tests.integration_tests.instances import IntegrationInstance -def _customize_envionment(client: IntegrationInstance): +def _customize_environment(client: IntegrationInstance): # Insert our "disable_network_activation" file here client.write_to_file( "/etc/cloud/cloud.cfg.d/99-disable-network-activation.cfg", @@ -19,7 +19,7 @@ def _customize_envionment(client: IntegrationInstance): @pytest.mark.ubuntu # Because netplan def test_network_activation_disabled(client: IntegrationInstance): """Test that the network is not activated during init mode.""" - _customize_envionment(client) + _customize_environment(client) result = client.execute("systemctl status google-guest-agent.service") if not result.ok: raise AssertionError( diff --git a/tests/integration_tests/datasources/test_oci_networking.py b/tests/integration_tests/datasources/test_oci_networking.py new file mode 100644 index 000000000..f569650e9 --- /dev/null +++ b/tests/integration_tests/datasources/test_oci_networking.py @@ -0,0 +1,118 @@ +import re +from typing import Iterator, Set + +import pytest +import yaml + +from tests.integration_tests.clouds import IntegrationCloud +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log + +DS_CFG = """\ +datasource: + Oracle: + configure_secondary_nics: {configure_secondary_nics} +""" + + +def customize_environment( + client: IntegrationInstance, + tmpdir, + configure_secondary_nics: bool = False, +): + cfg = tmpdir.join("01_oracle_datasource.cfg") + with open(cfg, "w") as f: + f.write( + DS_CFG.format(configure_secondary_nics=configure_secondary_nics) + ) + client.push_file(cfg, "/etc/cloud/cloud.cfg.d/01_oracle_datasource.cfg") + + client.execute("cloud-init clean --logs") + client.restart() + + +def extract_interface_names(network_config: dict) -> Set[str]: + if network_config["version"] == 1: + interfaces = map(lambda conf: conf["name"], network_config["config"]) + elif network_config["version"] == 2: + interfaces = network_config["ethernets"].keys() + else: + raise NotImplementedError( + f'Implement me for version={network_config["version"]}' + ) + return set(interfaces) + + +@pytest.mark.oci +def test_oci_networking_iscsi_instance(client: IntegrationInstance, tmpdir): + customize_environment(client, tmpdir, configure_secondary_nics=False) + result_net_files = client.execute("ls /run/net-*.conf") + assert result_net_files.ok, "No net files found under /run" + + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + + assert ( + "opc/v2/vnics/" not in log + ), "vnic data was fetched and it should not have been" + + netplan_yaml = client.read_from_file("/etc/netplan/50-cloud-init.yaml") + netplan_cfg = yaml.safe_load(netplan_yaml) + configured_interfaces = extract_interface_names(netplan_cfg["network"]) + assert 1 <= len( + configured_interfaces + ), "Expected at least 1 primary network configuration." + + expected_interfaces = set( + re.findall(r"/run/net-(.+)\.conf", result_net_files.stdout) + ) + for expected_interface in expected_interfaces: + assert ( + f"Reading from /run/net-{expected_interface}.conf" in log + ), "Expected {expected_interface} not found in: {log}" + + not_found_interfaces = expected_interfaces.difference( + configured_interfaces + ) + assert not not_found_interfaces, ( + f"Interfaces, {not_found_interfaces}, expected to be configured in" + f" {netplan_cfg['network']}" + ) + assert client.execute("ping -c 2 canonical.com").ok + + +@pytest.fixture(scope="function") +def client_with_secondary_vnic( + session_cloud: IntegrationCloud, +) -> Iterator[IntegrationInstance]: + """Create an instance client and attach a temporary vnic""" + with session_cloud.launch() as client: + ip_address = client.instance.add_network_interface() + yield client + client.instance.remove_network_interface(ip_address) + + +@pytest.mark.oci +def test_oci_networking_iscsi_instance_secondary_vnics( + client_with_secondary_vnic: IntegrationInstance, tmpdir +): + client = client_with_secondary_vnic + customize_environment(client, tmpdir, configure_secondary_nics=True) + + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + + assert "opc/v2/vnics/" in log, f"vnics data not fetched in {log}" + netplan_yaml = client.read_from_file("/etc/netplan/50-cloud-init.yaml") + netplan_cfg = yaml.safe_load(netplan_yaml) + configured_interfaces = extract_interface_names(netplan_cfg["network"]) + assert 2 <= len( + configured_interfaces + ), "Expected at least 1 primary and 1 secondary network configurations" + + result_net_files = client.execute("ls /run/net-*.conf") + expected_interfaces = set( + re.findall(r"/run/net-(.+)\.conf", result_net_files.stdout) + ) + assert len(expected_interfaces) + 1 == len(configured_interfaces) + assert client.execute("ping -c 2 canonical.com").ok diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index 65cd977a1..bd807cefe 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -60,6 +60,7 @@ def __init__( self.cloud = cloud self.instance = instance self.settings = settings + self._ip = "" def destroy(self): self.instance.delete() @@ -193,9 +194,20 @@ def upgrade_cloud_init(self): assert self.execute("apt-get update -q").ok assert self.execute("apt-get install -qy cloud-init").ok + def ip(self) -> str: + if self._ip: + return self._ip + try: + self._ip = self.instance.ip + except NotImplementedError: + self._ip = "Unknown" + return self._ip + def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): if not self.settings.KEEP_INSTANCE: self.destroy() + else: + log.info("Keeping Instance, public ip: %s", self.ip) diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index f27e4f12f..abc70fe4d 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import os +from typing import Optional from cloudinit.util import is_false, is_true @@ -26,7 +27,7 @@ # The cloud-specific instance type to run. E.g., a1.medium on AWS # If the pycloudlib instance provides a default, this can be left None -INSTANCE_TYPE = None +INSTANCE_TYPE: Optional[str] = None # Determines the base image to use or generate new images from. # @@ -38,7 +39,7 @@ # Populate if you want to use a pre-launched instance instead of # creating a new one. The exact contents will be platform dependent -EXISTING_INSTANCE_ID = None +EXISTING_INSTANCE_ID: Optional[str] = None ################################################################## # IMAGE GENERATION SETTINGS diff --git a/tests/integration_tests/modules/test_ansible.py b/tests/integration_tests/modules/test_ansible.py new file mode 100644 index 000000000..0d979d405 --- /dev/null +++ b/tests/integration_tests/modules/test_ansible.py @@ -0,0 +1,133 @@ +import pytest + +from tests.integration_tests.util import verify_clean_log + +# This works by setting up a local repository and web server +# daemon on the first boot. Second boot should succeed +# with the running web service and git repo configured. +# This instrumentation allows the test to run self-contained +# without network access or external git repos. + +REPO_D = "/root/playbooks" +USER_DATA = """\ +#cloud-config +version: v1 +packages_update: true +packages_upgrade: true +packages: + - git + - python3-pip +write_files: + - path: /etc/systemd/system/repo_server.service + content: | + [Unit] + Description=Serve a local git repo + Wants=repo_waiter.service + After=cloud-init-local.service + Before=cloud-config.service + Before=cloud-final.service + + [Install] + WantedBy=cloud-init-local.service + + [Service] + WorkingDirectory=/root/playbooks/.git + ExecStart=/usr/bin/env python3 -m http.server --bind 0.0.0.0 8000 + + + - path: /etc/systemd/system/repo_waiter.service + content: | + [Unit] + Description=Block boot until repo is available + After=repo_server.service + Before=cloud-final.service + + [Install] + WantedBy=cloud-init-local.service + + # clone into temp directory to test that server is running + # sdnotify would be an alternative way to verify that the server is + # running and continue once it is up, but this is simple and works + [Service] + Type=oneshot + ExecStart=/bin/sh -c "while \ + ! git clone http://0.0.0.0:8000/ $(mktemp -d); do sleep 0.1; done" + + - path: /root/playbooks/ubuntu.yml + content: | + --- + - hosts: 127.0.0.1 + connection: local + become: true + vars: + packages: + - git + - python3-pip + roles: + - apt + - path: /root/playbooks/roles/apt/tasks/main.yml + content: | + --- + - name: "install packages" + apt: + name: "*" + update_cache: yes + cache_valid_time: 3600 + - name: "install packages" + apt: + name: + - "{{ item }}" + state: latest + loop: "{{ packages }}" +runcmd: + - [systemctl, enable, repo_server.service] + - [systemctl, enable, repo_waiter.service] +""" + +INSTALL_METHOD = """ +ansible: + install-method: {method} + package-name: {package} + pull: + url: "http://0.0.0.0:8000/" + playbook-name: ubuntu.yml + full: true +""" + +SETUP_REPO = f"cd {REPO_D} &&\ +git config --global user.name auto &&\ +git config --global user.email autom@tic.io &&\ +git config --global init.defaultBranch main &&\ +git init {REPO_D} &&\ +git add {REPO_D}/roles/apt/tasks/main.yml {REPO_D}/ubuntu.yml &&\ +git commit -m auto &&\ +(cd {REPO_D}/.git; git update-server-info)" + + +def _test_ansible_pull_from_local_server(my_client): + setup = my_client.execute(SETUP_REPO) + assert not setup.stderr + assert not setup.return_code + my_client.execute("cloud-init clean --logs") + my_client.restart() + log = my_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + output_log = my_client.read_from_file("/var/log/cloud-init-output.log") + assert "ok=3" in output_log + assert "SUCCESS: config-ansible ran successfully" in log + + +@pytest.mark.user_data( + USER_DATA + INSTALL_METHOD.format(package="ansible-core", method="pip") +) +class TestAnsiblePullPip: + def test_ansible_pull_pip(self, class_client): + _test_ansible_pull_from_local_server(class_client) + + +@pytest.mark.user_data( + USER_DATA + INSTALL_METHOD.format(package="ansible", method="distro") +) +class TestAnsiblePullDistro: + def test_ansible_pull_distro(self, class_client): + _test_ansible_pull_from_local_server(class_client) diff --git a/tests/integration_tests/modules/test_ca_certs.py b/tests/integration_tests/modules/test_ca_certs.py index 7247fd7d6..8d18fb765 100644 --- a/tests/integration_tests/modules/test_ca_certs.py +++ b/tests/integration_tests/modules/test_ca_certs.py @@ -10,6 +10,9 @@ import pytest +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import get_inactive_modules, verify_clean_log + USER_DATA = """\ #cloud-config ca_certs: @@ -57,7 +60,7 @@ @pytest.mark.ubuntu @pytest.mark.user_data(USER_DATA) class TestCaCerts: - def test_certs_updated(self, class_client): + def test_certs_updated(self, class_client: IntegrationInstance): """Test that /etc/ssl/certs is updated as we expect.""" root = "/etc/ssl/certs" filenames = class_client.execute(["ls", "-1", root]).splitlines() @@ -79,7 +82,7 @@ def test_certs_updated(self, class_client): == links["cloud-init-ca-certs.pem"] ) - def test_cert_installed(self, class_client): + def test_cert_installed(self, class_client: IntegrationInstance): """Test that our specified cert has been installed""" checksum = class_client.execute( "sha256sum /etc/ssl/certs/ca-certificates.crt" @@ -88,3 +91,58 @@ def test_cert_installed(self, class_client): "78e875f18c73c1aab9167ae0bd323391e52222cc2dbcda42d129537219300062" in checksum ) + + def test_clean_log(self, class_client: IntegrationInstance): + """Verify no errors, no deprecations and correct inactive modules in + log. + """ + log = class_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log, ignore_deprecations=False) + + expected_inactive = { + "apt-pipelining", + "ansible", + "bootcmd", + "chef", + "disable-ec2-metadata", + "disk_setup", + "fan", + "keyboard", + "landscape", + "lxd", + "mcollective", + "ntp", + "package-update-upgrade-install", + "phone-home", + "power-state-change", + "puppet", + "rsyslog", + "runcmd", + "salt-minion", + "snap", + "timezone", + "ubuntu_autoinstall", + "ubuntu-advantage", + "ubuntu-drivers", + "update_etc_hosts", + "wireguard", + "write-files", + "write-files-deferred", + } + + # Remove modules that run independent from user-data + if class_client.settings.PLATFORM == "azure": + expected_inactive.discard("disk_setup") + elif class_client.settings.PLATFORM == "gce": + expected_inactive.discard("ntp") + elif class_client.settings.PLATFORM == "lxd_vm": + if class_client.settings.OS_IMAGE == "bionic": + expected_inactive.discard("write-files") + expected_inactive.discard("write-files-deferred") + + diff = expected_inactive.symmetric_difference( + get_inactive_modules(log) + ) + assert ( + not diff + ), f"Expected inactive modules do not match, diff: {diff}" diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 70850fd90..93523bfc4 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -19,6 +19,7 @@ from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.util import ( + get_inactive_modules, verify_clean_log, verify_ordered_items_in_text, ) @@ -66,8 +67,8 @@ commands: - snap install hello-world ssh_import_id: - - gh:powersj - lp:smoser + timezone: US/Aleutian """ @@ -173,7 +174,9 @@ def test_timezone(self, class_client: IntegrationInstance): assert timezone_output.strip() == "HDT" def test_no_problems(self, class_client: IntegrationInstance): - """Test no errors, warnings, or tracebacks""" + """Test no errors, warnings, deprecations, tracebacks or + inactive modules. + """ client = class_client status_file = client.read_from_file("/run/cloud-init/status.json") status_json = json.loads(status_file)["v1"] @@ -184,7 +187,26 @@ def test_no_problems(self, class_client: IntegrationInstance): assert result_json["errors"] == [] log = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) + verify_clean_log(log, ignore_deprecations=False) + requested_modules = { + "apt_configure", + "apt_pipelining", + "byobu", + "final_message", + "locale", + "ntp", + "seed_random", + "rsyslog", + "runcmd", + "snap", + "ssh_import_id", + "timezone", + } + inactive_modules = get_inactive_modules(log) + assert not requested_modules.intersection(inactive_modules), ( + f"Expected active modules:" + f" {requested_modules.intersection(inactive_modules)}" + ) def test_correct_datasource_detected( self, class_client: IntegrationInstance @@ -198,7 +220,6 @@ def test_correct_datasource_detected( if ImageSpecification.from_os_image().release in [ "bionic", "focal", - "impish", ]: datasource = "DataSourceNoCloud" else: @@ -276,7 +297,6 @@ def test_instance_json_lxd(self, class_client: IntegrationInstance): if ImageSpecification.from_os_image().release not in [ "bionic", "focal", - "impish", ]: cloud_name = "lxd" subplatform = "LXD socket API v. 1.0 (/dev/lxd/sock)" @@ -316,7 +336,6 @@ def test_instance_json_lxd_vm(self, class_client: IntegrationInstance): if ImageSpecification.from_os_image().release not in [ "bionic", "focal", - "impish", ]: cloud_name = "lxd" subplatform = "LXD socket API v. 1.0 (/dev/lxd/sock)" diff --git a/tests/integration_tests/modules/test_lxd.py b/tests/integration_tests/modules/test_lxd.py new file mode 100644 index 000000000..3443b74af --- /dev/null +++ b/tests/integration_tests/modules/test_lxd.py @@ -0,0 +1,88 @@ +"""Integration tests for LXD bridge creation. + +(This is ported from +``tests/cloud_tests/testcases/modules/lxd_bridge.yaml``.) +""" +import warnings + +import pytest +import yaml + +from tests.integration_tests.util import verify_clean_log + +BRIDGE_USER_DATA = """\ +#cloud-config +lxd: + init: + storage_backend: btrfs + bridge: + mode: new + name: lxdbr0 + ipv4_address: 10.100.100.1 + ipv4_netmask: 24 + ipv4_dhcp_first: 10.100.100.100 + ipv4_dhcp_last: 10.100.100.200 + ipv4_nat: true + domain: lxd + mtu: 9000 +""" + +STORAGE_USER_DATA = """\ +#cloud-config +lxd: + init: + storage_backend: {} +""" + + +@pytest.mark.no_container +@pytest.mark.user_data(BRIDGE_USER_DATA) +class TestLxdBridge: + @pytest.mark.parametrize("binary_name", ["lxc", "lxd"]) + def test_binaries_installed(self, class_client, binary_name): + """Check that the expected LXD binaries are installed""" + assert class_client.execute(["which", binary_name]).ok + + def test_bridge(self, class_client): + """Check that the given bridge is configured""" + cloud_init_log = class_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(cloud_init_log) + + # The bridge should exist + assert class_client.execute("ip addr show lxdbr0").ok + + raw_network_config = class_client.execute("lxc network show lxdbr0") + network_config = yaml.safe_load(raw_network_config) + assert "10.100.100.1/24" == network_config["config"]["ipv4.address"] + + +def validate_storage(validate_client, pkg_name, command): + log = validate_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log, ignore_deprecations=False) + return log + + +@pytest.mark.no_container +@pytest.mark.user_data(STORAGE_USER_DATA.format("btrfs")) +def test_storage_btrfs(client): + validate_storage(client, "btrfs-progs", "mkfs.btrfs") + + +@pytest.mark.no_container +@pytest.mark.user_data(STORAGE_USER_DATA.format("lvm")) +def test_storage_lvm(client): + log = client.read_from_file("/var/log/cloud-init.log") + + # Note to self + if ( + "doesn't use thinpool by default on Ubuntu due to LP" not in log + and "-kvm" not in client.execute("uname -r") + ): + warnings.warn("LP 1982780 has been fixed, update to allow thinpools") + validate_storage(client, "lvm2", "lvcreate") + + +@pytest.mark.no_container +@pytest.mark.user_data(STORAGE_USER_DATA.format("zfs")) +def test_storage_zfs(client): + validate_storage(client, "zfsutils-linux", "zpool") diff --git a/tests/integration_tests/modules/test_lxd_bridge.py b/tests/integration_tests/modules/test_lxd_bridge.py deleted file mode 100644 index 3292a833d..000000000 --- a/tests/integration_tests/modules/test_lxd_bridge.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Integration tests for LXD bridge creation. - -(This is ported from -``tests/cloud_tests/testcases/modules/lxd_bridge.yaml``.) -""" -import pytest -import yaml - -from tests.integration_tests.util import verify_clean_log - -USER_DATA = """\ -#cloud-config -lxd: - init: - storage_backend: dir - bridge: - mode: new - name: lxdbr0 - ipv4_address: 10.100.100.1 - ipv4_netmask: 24 - ipv4_dhcp_first: 10.100.100.100 - ipv4_dhcp_last: 10.100.100.200 - ipv4_nat: true - domain: lxd -""" - - -@pytest.mark.no_container -@pytest.mark.user_data(USER_DATA) -class TestLxdBridge: - @pytest.mark.parametrize("binary_name", ["lxc", "lxd"]) - def test_binaries_installed(self, class_client, binary_name): - """Check that the expected LXD binaries are installed""" - assert class_client.execute(["which", binary_name]).ok - - def test_bridge(self, class_client): - """Check that the given bridge is configured""" - cloud_init_log = class_client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(cloud_init_log) - - # The bridge should exist - assert class_client.execute("ip addr show lxdbr0") - - raw_network_config = class_client.execute("lxc network show lxdbr0") - network_config = yaml.safe_load(raw_network_config) - assert "10.100.100.1/24" == network_config["config"]["ipv4.address"] diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py index 66ea52dd9..4e0ee122e 100644 --- a/tests/integration_tests/modules/test_set_password.py +++ b/tests/integration_tests/modules/test_set_password.py @@ -11,6 +11,7 @@ import pytest import yaml +from tests.integration_tests.clouds import ImageSpecification from tests.integration_tests.decorators import retry from tests.integration_tests.util import get_console_log @@ -64,6 +65,23 @@ """ ) +USERS_USER_DATA = ( + COMMON_USER_DATA + + """ +chpasswd: + users: + - name: tom + password: mypassword123! + type: text + - name: dick + type: RANDOM + - name: harry + type: RANDOM + - name: mikey + password: $5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +""" +) + USERS_DICTS = yaml.safe_load(COMMON_USER_DATA)["users"] USERS_PASSWD_VALUES = { user_dict["name"]: user_dict["passwd"] @@ -160,22 +178,36 @@ def test_shadow_expected_users(self, class_client): shadow = class_client.read_from_file("/etc/shadow") for user_dict in USERS_DICTS: if "name" in user_dict: - assert "{}:".format(user_dict["name"]) in shadow + assert f'{user_dict["name"]}:' in shadow + + def test_sshd_config_file(self, class_client): + """Test that SSH config is written in the correct file.""" + if ImageSpecification.from_os_image().release in {"bionic"}: + sshd_file_target = "/etc/ssh/sshd_config" + else: + sshd_file_target = "/etc/ssh/sshd_config.d/50-cloud-init.conf" + assert class_client.execute(f"ls {sshd_file_target}").ok + sshd_config = class_client.read_from_file(sshd_file_target) + # We look for the exact line match, to avoid a commented line matching + assert "PasswordAuthentication yes" in sshd_config.splitlines() def test_sshd_config(self, class_client): """Test that SSH password auth is enabled.""" - sshd_config = class_client.read_from_file("/etc/ssh/sshd_config") - # We look for the exact line match, to avoid a commented line matching - assert "PasswordAuthentication yes" in sshd_config.splitlines() + sshd_config = class_client.execute("sshd -T").stdout + assert "passwordauthentication yes" in sshd_config -@pytest.mark.ci @pytest.mark.user_data(LIST_USER_DATA) class TestPasswordList(Mixin): """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass.""" -@pytest.mark.ci @pytest.mark.user_data(STRING_USER_DATA) class TestPasswordListString(Mixin): """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass.""" + + +@pytest.mark.ci +@pytest.mark.user_data(USERS_USER_DATA) +class TestPasswordUsersList(Mixin): + """Launch an instance with USERS_USER_DATA, ensure Mixin tests pass.""" diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py index b79f18ebe..8e73267aa 100644 --- a/tests/integration_tests/modules/test_ssh_keys_provided.py +++ b/tests/integration_tests/modules/test_ssh_keys_provided.py @@ -9,6 +9,8 @@ import pytest +from tests.integration_tests.clouds import ImageSpecification + USER_DATA = """\ #cloud-config disable_root: false @@ -109,10 +111,6 @@ class TestSshKeysProvided: "AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpg" "BP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAAD", ), - ( - "/etc/ssh/sshd_config", - "HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub", - ), ( "/etc/ssh/ssh_host_ecdsa_key.pub", "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAAB" @@ -138,3 +136,14 @@ class TestSshKeysProvided: def test_ssh_provided_keys(self, config_path, expected_out, class_client): out = class_client.read_from_file(config_path).strip() assert expected_out in out + + @pytest.mark.parametrize( + "expected_out", ("HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub") + ) + def test_sshd_config(self, expected_out, class_client): + if ImageSpecification.from_os_image().release in {"bionic"}: + sshd_config_path = "/etc/ssh/sshd_config" + else: + sshd_config_path = "/etc/ssh/sshd_config.d/50-cloud-init.conf" + sshd_config = class_client.read_from_file(sshd_config_path).strip() + assert expected_out in sshd_config diff --git a/tests/integration_tests/modules/test_ubuntu_autoinstall.py b/tests/integration_tests/modules/test_ubuntu_autoinstall.py new file mode 100644 index 000000000..d340afc56 --- /dev/null +++ b/tests/integration_tests/modules/test_ubuntu_autoinstall.py @@ -0,0 +1,26 @@ +"""Integration tests for cc_ubuntu_autoinstall happy path""" + +import pytest + +USER_DATA = """\ +#cloud-config +autoinstall: + version: 1 + cloudinitdoesnotvalidateotherkeyschema: true +snap: + commands: + - snap install subiquity --classic +""" + + +LOG_MSG = "Valid autoinstall schema. Config will be processed by subiquity" + + +@pytest.mark.ubuntu +@pytest.mark.user_data(USER_DATA) +class TestUbuntuAutoinstall: + def test_autoinstall_schema_valid_when_snap_present(self, class_client): + """autoinstall directives will pass when snap is present""" + assert "subiquity" in class_client.execute(["snap", "list"]).stdout + log = class_client.read_from_file("/var/log/cloud-init.log") + assert LOG_MSG in log diff --git a/tests/integration_tests/modules/test_ubuntu_drivers.py b/tests/integration_tests/modules/test_ubuntu_drivers.py new file mode 100644 index 000000000..4fbfba3c1 --- /dev/null +++ b/tests/integration_tests/modules/test_ubuntu_drivers.py @@ -0,0 +1,37 @@ +import re + +import pytest + +from tests.integration_tests.clouds import IntegrationCloud +from tests.integration_tests.util import verify_clean_log + +USER_DATA = """\ +#cloud-config +drivers: + nvidia: + license-accepted: true +""" + +# NOTE(VM.GPU2.1 is not in all availability_domains: use qIZq:US-ASHBURN-AD-1) + + +@pytest.mark.adhoc # Expensive instance type +@pytest.mark.oci +def test_ubuntu_drivers_installed(session_cloud: IntegrationCloud): + with session_cloud.launch( + launch_kwargs={"instance_type": "VM.GPU2.1"}, user_data=USER_DATA + ) as client: + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + assert 1 == log.count( + "Installing and activating NVIDIA drivers " + "(nvidia/license-accepted=True, version=latest)" + ) + result = client.execute("dpkg -l | grep nvidia") + assert result.ok, "No nvidia packages found" + assert re.search( + r"ii\s+linux-modules-nvidia-\d+-server", result.stdout + ), ( + f"Did not find specific nvidia drivers packages in:" + f" {result.stdout}" + ) diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 8fa37bb4e..91eca345a 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -10,6 +10,7 @@ from tests.integration_tests.clouds import ImageSpecification from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_log USER_DATA = """\ #cloud-config @@ -25,7 +26,7 @@ gecos: Foo B. Bar primary_group: foobar groups: users - expiredate: 2038-01-19 + expiredate: '2038-01-19' lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYe\ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -36,12 +37,13 @@ lock_passwd: true - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '0' system: true - name: eric + sudo: null uid: 1742 - name: archivist - uid: '1743' + uid: 1743 """ @@ -97,6 +99,8 @@ def test_users_groups(self, regex, getent_args, class_client): def test_user_root_in_secret(self, class_client): """Test root user is in 'secret' group.""" + log = class_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) output = class_client.execute("groups root").stdout _, groups_str = output.split(":", maxsplit=1) groups = groups_str.split() diff --git a/tests/integration_tests/modules/test_wireguard.py b/tests/integration_tests/modules/test_wireguard.py new file mode 100644 index 000000000..e658a9dff --- /dev/null +++ b/tests/integration_tests/modules/test_wireguard.py @@ -0,0 +1,129 @@ +"""Integration test for the wireguard module.""" +import pytest +from pycloudlib.lxd.instance import LXDInstance + +from cloudinit.subp import subp +from tests.integration_tests.instances import IntegrationInstance + +ASCII_TEXT = "ASCII text" + +USER_DATA = """\ +#cloud-config +wireguard: + interfaces: + - name: wg0 + config_path: /etc/wireguard/wg0.conf + content: | + [Interface] + Address = 192.168.254.1/32 + ListenPort = 51820 + PrivateKey = iNlmgtGo6yiFhD9TuVnx/qJSp+C5Cwg4wwPmOJwlZXI= + + [Peer] + PublicKey = 6PewunPjxlUq/0xvbVxklN2p73YIytfjxpoIEohCukY= + AllowedIPs = 192.168.254.2/32 + - name: wg1 + config_path: /etc/wireguard/wg1.conf + content: | + [Interface] + PrivateKey = GGLU4+5vIcK9lGyfz4AJn9fR5/FN/6sf4Fd5chZ16Vc= + Address = 192.168.254.2/24 + + [Peer] + PublicKey = 2as8z3EDjSsfFEkvOQGVnJ1Hv+h1jRAh2BKJg+DHvGk= + Endpoint = 127.0.0.1:51820 + AllowedIPs = 0.0.0.0/0 + readinessprobe: + - ping -qc 5 192.168.254.1 2>&1 > /dev/null + - echo $? > /tmp/ping + +# wg-quick configures the system interfaces and routes, but we need to ssh in +# stop the service at the end of cloud-init +runcmd: + - [systemctl, stop, wg-quick@wg0.service] + - [systemctl, stop, wg-quick@wg1.service] +""" + + +def load_wireguard_kernel_module_lxd(instance: LXDInstance): + subp( + "lxc config set {} linux.kernel_modules wireguard".format( + instance.name + ).split() + ) + + +@pytest.mark.ci +@pytest.mark.user_data(USER_DATA) +@pytest.mark.lxd_vm +@pytest.mark.gce +@pytest.mark.ec2 +@pytest.mark.azure +@pytest.mark.openstack +@pytest.mark.oci +@pytest.mark.ubuntu +class TestWireguard: + @pytest.mark.parametrize( + "cmd,expected_out", + ( + # check if wireguard module is loaded + ("lsmod | grep '^wireguard' | awk '{print $1}'", "wireguard"), + # test if file was written for wg0 + ( + "stat -c '%N' /etc/wireguard/wg0.conf", + r"'/etc/wireguard/wg0.conf'", + ), + # check permissions for wg0 + ("stat -c '%U %a' /etc/wireguard/wg0.conf", r"root 600"), + # ASCII check wg1 + ("file /etc/wireguard/wg1.conf", ASCII_TEXT), + # md5sum check wg1 + ( + "md5sum Iterator[IntegrationInstance]: + client.write_to_file( + f"/etc/cloud/cloud.cfg.d/{CUSTOM_CLOUD_DIR_FN}", CUSTOM_CLOUD_DIR + ) + client.execute(f"rm -rf {DEFAULT_CLOUD_DIR}") # Remove previous cloud_dir + client.execute("cloud-init clean --logs") + client.restart() + yield client + + +class TestHonorCloudDir: + def verify_log_and_files(self, custom_client): + log_content = custom_client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log_content) + assert NEW_CLOUD_DIR in log_content + assert DEFAULT_CLOUD_DIR not in log_content + assert custom_client.execute(f"test ! -d {DEFAULT_CLOUD_DIR}").ok + + def collect_logs(self, custom_client: IntegrationInstance): + help_result = custom_client.execute("cloud-init collect-logs -h") + assert help_result.ok, help_result.stderr + assert f"{NEW_CLOUD_DIR}/instance/user-data.txt" in re.sub( + r"\s+", "", help_result.stdout + ), "user-data file not correctly render in collect-logs -h" + collect_logs_result = custom_client.execute( + "cloud-init collect-logs --include-userdata" + ) + assert ( + collect_logs_result.ok + ), f"collect-logs error: {collect_logs_result.stderr}" + + # LXD inserts some agent setup code into VMs on Bionic under + # /var/lib/cloud. The inserted script will cause this test to fail + # because the test ensures nothing is running under /var/lib/cloud. + # Since LXD is doing this and not cloud-init, we should just not run + # on Bionic to avoid it. + @pytest.mark.not_bionic + def test_honor_cloud_dir(self, custom_client: IntegrationInstance): + """Integration test for LP: #1976564 + + cloud-init must honor the cloud-dir configured in + /etc/cloud/cloud.cfg.d + """ + self.verify_log_and_files(custom_client) + self.collect_logs(custom_client) diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index b13d4703d..5ef82e888 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -95,7 +95,12 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud): # have broken across re-constitution of a cached datasource. Some # platforms invalidate their datasource cache on reboot, so we run # it here to ensure we get a dirty run. - assert instance.execute("cloud-init init").ok + assert instance.execute( + "cloud-init init --local; " + "cloud-init init; " + "cloud-init modules --mode=config; " + "cloud-init modules --mode=final" + ).ok # Reboot instance.execute("hostname something-else") @@ -185,4 +190,6 @@ def test_subsequent_boot_of_upgraded_package(session_cloud: IntegrationCloud): source, take_snapshot=False, clean=False ) instance.restart() + log = instance.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) assert instance.execute("cloud-init status --wait --long").ok diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index ec6b14347..f955e94ec 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -5,7 +5,9 @@ import time from collections import namedtuple from contextlib import contextmanager +from itertools import chain from pathlib import Path +from typing import Set import pytest @@ -35,8 +37,22 @@ def verify_ordered_items_in_text(to_verify: list, text: str): index = matched.start() -def verify_clean_log(log): +def verify_clean_log(log: str, ignore_deprecations: bool = True): """Assert no unexpected tracebacks or warnings in logs""" + if ignore_deprecations: + is_deprecated = re.compile("deprecat", flags=re.IGNORECASE) + log_lines = log.split("\n") + log_lines = list( + filter(lambda line: not is_deprecated.search(line), log_lines) + ) + log = "\n".join(log_lines) + + error_logs = re.findall("ERROR.*", log) + if error_logs: + raise AssertionError( + "Found unexpected errors: %s" % "\n".join(error_logs) + ) + warning_count = log.count("WARN") expected_warnings = 0 traceback_count = log.count("Traceback") @@ -45,7 +61,10 @@ def verify_clean_log(log): warning_texts = [ # Consistently on all Azure launches: # azure.py[WARNING]: No lease found; using default endpoint - "No lease found; using default endpoint" + "No lease found; using default endpoint", + # Ubuntu lxd storage + "thinpool by default on Ubuntu due to LP #1982780", + "WARNING]: Could not match supplied host pattern, ignoring:", ] traceback_texts = [] if "oracle" in log: @@ -82,6 +101,19 @@ def verify_clean_log(log): assert traceback_count == expected_tracebacks +def get_inactive_modules(log: str) -> Set[str]: + matches = re.findall( + r"Skipping modules '(.*)' because no applicable config is provided.", + log, + ) + return set( + map( + lambda module: module.strip(), + chain(*map(lambda match: match.split(","), matches)), + ) + ) + + @contextmanager def emit_dots_on_travis(): """emit a dot every 60 seconds if running on Travis. diff --git a/tests/unittests/analyze/test_boot.py b/tests/unittests/analyze/test_boot.py index 68db69ec6..261f4c4e9 100644 --- a/tests/unittests/analyze/test_boot.py +++ b/tests/unittests/analyze/test_boot.py @@ -112,19 +112,19 @@ def test_boot_invalid_distro(self, m_dist_check_timestamp): analyze_boot(name_default, args) # now args have been tested, go into outfile and make sure error # message is in the outfile - outfh = open(args.outfile, "r") - data = outfh.read() - err_string = ( - "Your Linux distro or container does not support this " - "functionality.\nYou must be running a Kernel " - "Telemetry supported distro.\nPlease check " - "https://cloudinit.readthedocs.io/en/latest/topics" - "/analyze.html for more information on supported " - "distros.\n" - ) - - self.remove_dummy_file(path, log_path) - self.assertEqual(err_string, data) + with open(args.outfile, "r") as outfh: + data = outfh.read() + err_string = ( + "Your Linux distro or container does not support this " + "functionality.\nYou must be running a Kernel " + "Telemetry supported distro.\nPlease check " + "https://cloudinit.readthedocs.io/en/latest/topics" + "/analyze.html for more information on supported " + "distros.\n" + ) + + self.remove_dummy_file(path, log_path) + self.assertEqual(err_string, data) @mock.patch("cloudinit.util.is_container", return_value=True) @mock.patch("cloudinit.subp.subp", return_value=("U=1000000", None)) diff --git a/tests/unittests/analyze/test_dump.py b/tests/unittests/analyze/test_dump.py index 56bbf97f9..1b4ce8204 100644 --- a/tests/unittests/analyze/test_dump.py +++ b/tests/unittests/analyze/test_dump.py @@ -216,8 +216,8 @@ def test_dump_events_with_cisource(self, m_parse_from_date): tmpfile = self.tmp_path("logfile") write_file(tmpfile, SAMPLE_LOGS) m_parse_from_date.return_value = 1472594005.972 - - events, data = dump_events(cisource=open(tmpfile)) + with open(tmpfile) as file: + events, data = dump_events(cisource=file) year = datetime.now().year dt1 = datetime.strptime( "Nov 03 06:51:06.074410 %d" % year, "%b %d %H:%M:%S.%f %Y" diff --git a/tests/unittests/cmd/devel/test_hotplug_hook.py b/tests/unittests/cmd/devel/test_hotplug_hook.py index 5ecb59699..d2ef82b15 100644 --- a/tests/unittests/cmd/devel/test_hotplug_hook.py +++ b/tests/unittests/cmd/devel/test_hotplug_hook.py @@ -19,7 +19,9 @@ @pytest.fixture def mocks(): m_init = mock.MagicMock(spec=Init) + m_activator = mock.MagicMock(spec=NetworkActivator) m_distro = mock.MagicMock(spec=Distro) + m_distro.network_activator = mock.PropertyMock(return_value=m_activator) m_datasource = mock.MagicMock(spec=DataSource) m_datasource.distro = m_distro m_init.datasource = m_datasource @@ -41,18 +43,11 @@ def mocks(): return_value=m_network_state, ) - m_activator = mock.MagicMock(spec=NetworkActivator) - select_activator = mock.patch( - "cloudinit.cmd.devel.hotplug_hook.activators.select_activator", - return_value=m_activator, - ) - sleep = mock.patch("time.sleep") read_sys_net.start() update_event_enabled.start() parse_net.start() - select_activator.start() m_sleep = sleep.start() yield namedtuple("mocks", "m_init m_network_state m_activator m_sleep")( @@ -65,7 +60,6 @@ def mocks(): read_sys_net.stop() update_event_enabled.stop() parse_net.stop() - select_activator.stop() sleep.stop() diff --git a/tests/unittests/cmd/devel/test_logs.py b/tests/unittests/cmd/devel/test_logs.py index 73ed3c656..c916c19f9 100644 --- a/tests/unittests/cmd/devel/test_logs.py +++ b/tests/unittests/cmd/devel/test_logs.py @@ -1,55 +1,52 @@ # This file is part of cloud-init. See LICENSE file for license information. import os +import re from datetime import datetime from io import StringIO from cloudinit.cmd.devel import logs from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.subp import subp -from cloudinit.util import ensure_dir, load_file, write_file -from tests.unittests.helpers import ( - FilesystemMockingTestCase, - mock, - wrap_and_call, -) +from cloudinit.util import load_file, write_file +from tests.unittests.helpers import mock +M_PATH = "cloudinit.cmd.devel.logs." -@mock.patch("cloudinit.cmd.devel.logs.os.getuid") -class TestCollectLogs(FilesystemMockingTestCase): - def setUp(self): - super(TestCollectLogs, self).setUp() - self.new_root = self.tmp_dir() - self.run_dir = self.tmp_path("run", self.new_root) - def test_collect_logs_with_userdata_requires_root_user(self, m_getuid): +@mock.patch("cloudinit.cmd.devel.logs.os.getuid") +class TestCollectLogs: + def test_collect_logs_with_userdata_requires_root_user( + self, m_getuid, tmpdir + ): """collect-logs errors when non-root user collects userdata .""" m_getuid.return_value = 100 # non-root - output_tarfile = self.tmp_path("logs.tgz") + output_tarfile = tmpdir.join("logs.tgz") with mock.patch("sys.stderr", new_callable=StringIO) as m_stderr: - self.assertEqual( - 1, logs.collect_logs(output_tarfile, include_userdata=True) + assert 1 == logs.collect_logs( + output_tarfile, include_userdata=True ) - self.assertEqual( + assert ( "To include userdata, root user is required." - " Try sudo cloud-init collect-logs\n", - m_stderr.getvalue(), + " Try sudo cloud-init collect-logs\n" == m_stderr.getvalue() ) - def test_collect_logs_creates_tarfile(self, m_getuid): + def test_collect_logs_creates_tarfile(self, m_getuid, mocker, tmpdir): """collect-logs creates a tarfile with all related cloud-init info.""" m_getuid.return_value = 100 - log1 = self.tmp_path("cloud-init.log", self.new_root) + log1 = tmpdir.join("cloud-init.log") write_file(log1, "cloud-init-log") - log2 = self.tmp_path("cloud-init-output.log", self.new_root) + log2 = tmpdir.join("cloud-init-output.log") write_file(log2, "cloud-init-output-log") - ensure_dir(self.run_dir) - write_file(self.tmp_path("results.json", self.run_dir), "results") + run_dir = tmpdir.join("run") + write_file(run_dir.join("results.json"), "results") write_file( - self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, self.run_dir), + run_dir.join( + INSTANCE_JSON_SENSITIVE_FILE, + ), "sensitive", ) - output_tarfile = self.tmp_path("logs.tgz") + output_tarfile = str(tmpdir.join("logs.tgz")) date = datetime.utcnow().date().strftime("%Y-%m-%d") date_logdir = "cloud-init-logs-{0}".format(date) @@ -80,76 +77,63 @@ def fake_subp(cmd): fake_stderr = mock.MagicMock() - wrap_and_call( - "cloudinit.cmd.devel.logs", - { - "subp": {"side_effect": fake_subp}, - "sys.stderr": {"new": fake_stderr}, - "CLOUDINIT_LOGS": {"new": [log1, log2]}, - "CLOUDINIT_RUN_DIR": {"new": self.run_dir}, - }, - logs.collect_logs, - output_tarfile, - include_userdata=False, - ) + mocker.patch(M_PATH + "subp", side_effect=fake_subp) + mocker.patch(M_PATH + "sys.stderr", fake_stderr) + mocker.patch(M_PATH + "CLOUDINIT_LOGS", [log1, log2]) + mocker.patch(M_PATH + "CLOUDINIT_RUN_DIR", run_dir) + logs.collect_logs(output_tarfile, include_userdata=False) # unpack the tarfile and check file contents - subp(["tar", "zxvf", output_tarfile, "-C", self.new_root]) - out_logdir = self.tmp_path(date_logdir, self.new_root) - self.assertFalse( - os.path.exists( - os.path.join( - out_logdir, - "run", - "cloud-init", - INSTANCE_JSON_SENSITIVE_FILE, - ) - ), - "Unexpected file found: %s" % INSTANCE_JSON_SENSITIVE_FILE, - ) - self.assertEqual( - "0.7fake\n", load_file(os.path.join(out_logdir, "dpkg-version")) + subp(["tar", "zxvf", output_tarfile, "-C", str(tmpdir)]) + out_logdir = tmpdir.join(date_logdir) + assert not os.path.exists( + os.path.join( + out_logdir, + "run", + "cloud-init", + INSTANCE_JSON_SENSITIVE_FILE, + ) + ), ( + "Unexpected file found: %s" % INSTANCE_JSON_SENSITIVE_FILE ) - self.assertEqual( - version_out, load_file(os.path.join(out_logdir, "version")) + assert "0.7fake\n" == load_file( + os.path.join(out_logdir, "dpkg-version") ) - self.assertEqual( - "cloud-init-log", - load_file(os.path.join(out_logdir, "cloud-init.log")), + assert version_out == load_file(os.path.join(out_logdir, "version")) + assert "cloud-init-log" == load_file( + os.path.join(out_logdir, "cloud-init.log") ) - self.assertEqual( - "cloud-init-output-log", - load_file(os.path.join(out_logdir, "cloud-init-output.log")), + assert "cloud-init-output-log" == load_file( + os.path.join(out_logdir, "cloud-init-output.log") ) - self.assertEqual( - "dmesg-out\n", load_file(os.path.join(out_logdir, "dmesg.txt")) + assert "dmesg-out\n" == load_file( + os.path.join(out_logdir, "dmesg.txt") ) - self.assertEqual( - "journal-out\n", load_file(os.path.join(out_logdir, "journal.txt")) + assert "journal-out\n" == load_file( + os.path.join(out_logdir, "journal.txt") ) - self.assertEqual( - "results", - load_file( - os.path.join(out_logdir, "run", "cloud-init", "results.json") - ), + assert "results" == load_file( + os.path.join(out_logdir, "run", "cloud-init", "results.json") ) fake_stderr.write.assert_any_call("Wrote %s\n" % output_tarfile) - def test_collect_logs_includes_optional_userdata(self, m_getuid): + def test_collect_logs_includes_optional_userdata( + self, m_getuid, mocker, tmpdir + ): """collect-logs include userdata when --include-userdata is set.""" m_getuid.return_value = 0 - log1 = self.tmp_path("cloud-init.log", self.new_root) + log1 = tmpdir.join("cloud-init.log") write_file(log1, "cloud-init-log") - log2 = self.tmp_path("cloud-init-output.log", self.new_root) + log2 = tmpdir.join("cloud-init-output.log") write_file(log2, "cloud-init-output-log") - userdata = self.tmp_path("user-data.txt", self.new_root) + userdata = tmpdir.join("user-data.txt") write_file(userdata, "user-data") - ensure_dir(self.run_dir) - write_file(self.tmp_path("results.json", self.run_dir), "results") + run_dir = tmpdir.join("run") + write_file(run_dir.join("results.json"), "results") write_file( - self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, self.run_dir), + run_dir.join(INSTANCE_JSON_SENSITIVE_FILE), "sensitive", ) - output_tarfile = self.tmp_path("logs.tgz") + output_tarfile = str(tmpdir.join("logs.tgz")) date = datetime.utcnow().date().strftime("%Y-%m-%d") date_logdir = "cloud-init-logs-{0}".format(date) @@ -180,34 +164,31 @@ def fake_subp(cmd): fake_stderr = mock.MagicMock() - wrap_and_call( - "cloudinit.cmd.devel.logs", - { - "subp": {"side_effect": fake_subp}, - "sys.stderr": {"new": fake_stderr}, - "CLOUDINIT_LOGS": {"new": [log1, log2]}, - "CLOUDINIT_RUN_DIR": {"new": self.run_dir}, - "USER_DATA_FILE": {"new": userdata}, - }, - logs.collect_logs, - output_tarfile, - include_userdata=True, - ) + mocker.patch(M_PATH + "subp", side_effect=fake_subp) + mocker.patch(M_PATH + "sys.stderr", fake_stderr) + mocker.patch(M_PATH + "CLOUDINIT_LOGS", [log1, log2]) + mocker.patch(M_PATH + "CLOUDINIT_RUN_DIR", run_dir) + mocker.patch(M_PATH + "_get_user_data_file", return_value=userdata) + logs.collect_logs(output_tarfile, include_userdata=True) # unpack the tarfile and check file contents - subp(["tar", "zxvf", output_tarfile, "-C", self.new_root]) - out_logdir = self.tmp_path(date_logdir, self.new_root) - self.assertEqual( - "user-data", load_file(os.path.join(out_logdir, "user-data.txt")) + subp(["tar", "zxvf", output_tarfile, "-C", str(tmpdir)]) + out_logdir = tmpdir.join(date_logdir) + assert "user-data" == load_file( + os.path.join(out_logdir, "user-data.txt") ) - self.assertEqual( - "sensitive", - load_file( - os.path.join( - out_logdir, - "run", - "cloud-init", - INSTANCE_JSON_SENSITIVE_FILE, - ) - ), + assert "sensitive" == load_file( + os.path.join( + out_logdir, + "run", + "cloud-init", + INSTANCE_JSON_SENSITIVE_FILE, + ) ) fake_stderr.write.assert_any_call("Wrote %s\n" % output_tarfile) + + +class TestParser: + def test_parser_help_has_userdata_file(self, mocker, tmpdir): + userdata = str(tmpdir.join("user-data.txt")) + mocker.patch(M_PATH + "_get_user_data_file", return_value=userdata) + assert userdata in re.sub(r"\s+", "", logs.get_parser().format_help()) diff --git a/tests/unittests/cmd/devel/test_net_convert.py b/tests/unittests/cmd/devel/test_net_convert.py new file mode 100644 index 000000000..100aa8de9 --- /dev/null +++ b/tests/unittests/cmd/devel/test_net_convert.py @@ -0,0 +1,229 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import itertools + +import pytest + +from cloudinit import safeyaml as yaml +from cloudinit.cmd.devel import net_convert +from cloudinit.distros.debian import NETWORK_FILE_HEADER +from tests.unittests.helpers import mock + +M_PATH = "cloudinit.cmd.devel.net_convert." + + +required_args = [ + "--directory", + "--network-data", + "--distro=ubuntu", + "--kind=eni", + "--output-kind=eni", +] + + +SAMPLE_NET_V1 = """\ +network: + version: 1 + config: + - type: physical + name: eth0 + subnets: + - type: dhcp +""" + + +SAMPLE_NETPLAN_CONTENT = f"""\ +{NETWORK_FILE_HEADER}network: + version: 2 + ethernets: + eth0: + dhcp4: true +""" + +SAMPLE_ENI_CONTENT = f"""\ +{NETWORK_FILE_HEADER}auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp +""" + +SAMPLE_NETWORKD_CONTENT = """\ +[Match] +Name=eth0 + +[Network] +DHCP=ipv4 + +""" + +SAMPLE_SYSCONFIG_CONTENT = """\ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""" + +SAMPLE_NETWORK_MANAGER_CONTENT = """\ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init eth0 +uuid=1dd9a779-d327-56e1-8454-c65e2556c12c +type=ethernet +interface-name=eth0 + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] + +[ipv4] +method=auto +may-fail=false + +""" + + +class TestNetConvert: + + missing_required_args = itertools.combinations( + required_args, len(required_args) - 1 + ) + + def _replace_path_args(self, cmd, tmpdir): + """Inject tmpdir replacements for parameterize args.""" + updated_cmd = [] + for arg in cmd: + if arg == "--network-data": + net_file = tmpdir.join("net") + net_file.write("") + updated_cmd.append(f"--network-data={net_file}") + elif arg == "--directory": + updated_cmd.append(f"--directory={tmpdir.strpath}") + else: + updated_cmd.append(arg) + return updated_cmd + + @pytest.mark.parametrize("cmdargs", missing_required_args) + def test_argparse_error_on_missing_args(self, cmdargs, capsys, tmpdir): + """Log the appropriate error when required args are missing.""" + params = self._replace_path_args(cmdargs, tmpdir) + with mock.patch("sys.argv", ["net-convert"] + params): + with pytest.raises(SystemExit): + net_convert.get_parser().parse_args() + _out, err = capsys.readouterr() + assert "the following arguments are required" in err + + @pytest.mark.parametrize("debug", (False, True)) + @pytest.mark.parametrize( + "output_kind,outfile_content", + ( + ( + "netplan", + {"etc/netplan/50-cloud-init.yaml": SAMPLE_NETPLAN_CONTENT}, + ), + ( + "eni", + { + "etc/network/interfaces.d/50-cloud-init.cfg": SAMPLE_ENI_CONTENT # noqa: E501 + }, + ), + ( + "networkd", + { + "etc/systemd/network/10-cloud-init-eth0.network": SAMPLE_NETWORKD_CONTENT # noqa: E501 + }, + ), + ( + "sysconfig", + { + "etc/sysconfig/network-scripts/ifcfg-eth0": SAMPLE_SYSCONFIG_CONTENT # noqa: E501 + }, + ), + ( + "network-manager", + { + "etc/NetworkManager/system-connections/cloud-init-eth0.nmconnection": SAMPLE_NETWORK_MANAGER_CONTENT # noqa: E501 + }, + ), + ), + ) + def test_convert_output_kind_artifacts( + self, output_kind, outfile_content, debug, capsys, tmpdir + ): + """Assert proper output-kind artifacts are written.""" + network_data = tmpdir.join("network_data") + network_data.write(SAMPLE_NET_V1) + distro = "centos" if output_kind == "sysconfig" else "ubuntu" + args = [ + f"--directory={tmpdir.strpath}", + f"--network-data={network_data.strpath}", + f"--distro={distro}", + "--kind=yaml", + f"--output-kind={output_kind}", + ] + if debug: + args.append("--debug") + params = self._replace_path_args(args, tmpdir) + with mock.patch("sys.argv", ["net-convert"] + params): + args = net_convert.get_parser().parse_args() + with mock.patch("cloudinit.util.chownbyname") as chown: + net_convert.handle_args("somename", args) + for path in outfile_content: + outfile = tmpdir.join(path) + assert outfile_content[path] == outfile.read() + if output_kind == "networkd": + assert [ + mock.call( + outfile.strpath, "systemd-network", "systemd-network" + ) + ] == chown.call_args_list + + @pytest.mark.parametrize("debug", (False, True)) + def test_convert_netplan_passthrough(self, debug, tmpdir): + """Assert that if the network config's version is 2 and the renderer is + Netplan, then the config is passed through as-is. + """ + network_data = tmpdir.join("network_data") + # `default` as a route supported by Netplan but not by cloud-init + content = """\ + network: + version: 2 + ethernets: + enp0s3: + dhcp4: false + addresses: [10.0.4.10/24] + nameservers: + addresses: [10.0.4.1] + routes: + - to: default + via: 10.0.4.1 + metric: 100 + """ + network_data.write(content) + args = [ + "-m", + "enp0s3,AA", + f"--directory={tmpdir.strpath}", + f"--network-data={network_data.strpath}", + "--distro=ubuntu", + "--kind=yaml", + "--output-kind=netplan", + ] + if debug: + args.append("--debug") + params = self._replace_path_args(args, tmpdir) + with mock.patch("sys.argv", ["net-convert"] + params): + args = net_convert.get_parser().parse_args() + with mock.patch("cloudinit.util.chownbyname"): + net_convert.handle_args("somename", args) + outfile = tmpdir.join("etc/netplan/50-cloud-init.yaml") + assert yaml.load(content) == yaml.load(outfile.read()) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/cmd/test_clean.py b/tests/unittests/cmd/test_clean.py index 7d12017e6..b859b83b1 100644 --- a/tests/unittests/cmd/test_clean.py +++ b/tests/unittests/cmd/test_clean.py @@ -2,170 +2,215 @@ import os from collections import namedtuple -from io import StringIO +import pytest + +import cloudinit.settings from cloudinit.cmd import clean -from cloudinit.util import ensure_dir, sym_link, write_file -from tests.unittests.helpers import CiTestCase, mock, wrap_and_call +from cloudinit.util import ensure_dir, sym_link +from tests.unittests.helpers import mock, wrap_and_call + +MyPaths = namedtuple("MyPaths", "cloud_dir") +CleanPaths = namedtuple( + "CleanPaths", ["tmpdir", "cloud_dir", "clean_dir", "log", "output_log"] +) + -mypaths = namedtuple("MyPaths", "cloud_dir") +@pytest.fixture(scope="function") +def clean_paths(tmpdir): + return CleanPaths( + tmpdir=tmpdir, + cloud_dir=tmpdir.join("varlibcloud"), + clean_dir=tmpdir.join("clean.d"), + log=tmpdir.join("cloud-init.log"), + output_log=tmpdir.join("cloud-init-output.log"), + ) -class TestClean(CiTestCase): - def setUp(self): - super(TestClean, self).setUp() - self.new_root = self.tmp_dir() - self.artifact_dir = self.tmp_path("artifacts", self.new_root) - self.log1 = self.tmp_path("cloud-init.log", self.new_root) - self.log2 = self.tmp_path("cloud-init-output.log", self.new_root) +@pytest.fixture(scope="function") +def init_class(clean_paths): + class FakeInit(object): + cfg = { + "def_log_file": clean_paths.log, + "output": {"all": f"|tee -a {clean_paths.output_log}"}, + } + # Ensure cloud_dir has a trailing slash, to match real behaviour + paths = MyPaths(cloud_dir=f"{clean_paths.cloud_dir}/") - class FakeInit(object): - cfg = { - "def_log_file": self.log1, - "output": {"all": "|tee -a {0}".format(self.log2)}, - } - # Ensure cloud_dir has a trailing slash, to match real behaviour - paths = mypaths(cloud_dir="{}/".format(self.artifact_dir)) + def __init__(self, ds_deps): + pass - def __init__(self, ds_deps): - pass + def read_cfg(self): + pass - def read_cfg(self): - pass + return FakeInit - self.init_class = FakeInit - def test_remove_artifacts_removes_logs(self): +class TestClean: + def test_remove_artifacts_removes_logs(self, clean_paths, init_class): """remove_artifacts removes logs when remove_logs is True.""" - write_file(self.log1, "cloud-init-log") - write_file(self.log2, "cloud-init-output-log") + clean_paths.log.write("cloud-init-log") + clean_paths.output_log.write("cloud-init-output-log") - self.assertFalse( - os.path.exists(self.artifact_dir), "Unexpected artifacts dir" - ) + assert ( + os.path.exists(clean_paths.cloud_dir) is False + ), "Unexpected cloud_dir" retcode = wrap_and_call( "cloudinit.cmd.clean", - {"Init": {"side_effect": self.init_class}}, + {"Init": {"side_effect": init_class}}, clean.remove_artifacts, remove_logs=True, ) - self.assertFalse(os.path.exists(self.log1), "Unexpected file") - self.assertFalse(os.path.exists(self.log2), "Unexpected file") - self.assertEqual(0, retcode) + assert ( + clean_paths.log.exists() is False + ), f"Unexpected file {clean_paths.log}" + assert ( + clean_paths.output_log.exists() is False + ), f"Unexpected file {clean_paths.output_log}" + assert 0 == retcode + + @pytest.mark.allow_all_subp + def test_remove_artifacts_runparts_clean_d(self, clean_paths, init_class): + """remove_artifacts performs runparts on CLEAN_RUNPARTS_DIR""" + ensure_dir(clean_paths.cloud_dir) + artifact_file = clean_paths.tmpdir.join("didit") + ensure_dir(clean_paths.clean_dir) + assert artifact_file.exists() is False, f"Unexpected {artifact_file}" + clean_script = clean_paths.clean_dir.join("1.sh") + clean_script.write(f"#!/bin/bash\ntouch {artifact_file}\n") + clean_script.chmod(mode=0o755) + with mock.patch.object( + cloudinit.settings, "CLEAN_RUNPARTS_DIR", clean_paths.clean_dir + ): + retcode = wrap_and_call( + "cloudinit.cmd.clean", + { + "Init": {"side_effect": init_class}, + }, + clean.remove_artifacts, + remove_logs=False, + ) + assert ( + artifact_file.exists() is True + ), f"Missing expected {artifact_file}" + assert 0 == retcode - def test_remove_artifacts_preserves_logs(self): + def test_remove_artifacts_preserves_logs(self, clean_paths, init_class): """remove_artifacts leaves logs when remove_logs is False.""" - write_file(self.log1, "cloud-init-log") - write_file(self.log2, "cloud-init-output-log") + clean_paths.log.write("cloud-init-log") + clean_paths.output_log.write("cloud-init-output-log") retcode = wrap_and_call( "cloudinit.cmd.clean", - {"Init": {"side_effect": self.init_class}}, + {"Init": {"side_effect": init_class}}, clean.remove_artifacts, remove_logs=False, ) - self.assertTrue(os.path.exists(self.log1), "Missing expected file") - self.assertTrue(os.path.exists(self.log2), "Missing expected file") - self.assertEqual(0, retcode) + assert 0 == retcode + assert ( + clean_paths.log.exists() is True + ), f"Missing expected file {clean_paths.log}" + assert ( + clean_paths.output_log.exists() + ), f"Missing expected file {clean_paths.output_log}" - def test_remove_artifacts_removes_unlinks_symlinks(self): + def test_remove_artifacts_removes_unlinks_symlinks( + self, clean_paths, init_class + ): """remove_artifacts cleans artifacts dir unlinking any symlinks.""" - dir1 = os.path.join(self.artifact_dir, "dir1") + dir1 = clean_paths.cloud_dir.join("dir1") ensure_dir(dir1) - symlink = os.path.join(self.artifact_dir, "mylink") - sym_link(dir1, symlink) + symlink = clean_paths.cloud_dir.join("mylink") + sym_link(dir1.strpath, symlink.strpath) retcode = wrap_and_call( "cloudinit.cmd.clean", - {"Init": {"side_effect": self.init_class}}, + {"Init": {"side_effect": init_class}}, clean.remove_artifacts, remove_logs=False, ) - self.assertEqual(0, retcode) + assert 0 == retcode for path in (dir1, symlink): - self.assertFalse( - os.path.exists(path), "Unexpected {0} dir".format(path) - ) + assert path.exists() is False, f"Unexpected {path} found" - def test_remove_artifacts_removes_artifacts_skipping_seed(self): + def test_remove_artifacts_removes_artifacts_skipping_seed( + self, clean_paths, init_class + ): """remove_artifacts cleans artifacts dir with exception of seed dir.""" dirs = [ - self.artifact_dir, - os.path.join(self.artifact_dir, "seed"), - os.path.join(self.artifact_dir, "dir1"), - os.path.join(self.artifact_dir, "dir2"), + clean_paths.cloud_dir, + clean_paths.cloud_dir.join("seed"), + clean_paths.cloud_dir.join("dir1"), + clean_paths.cloud_dir.join("dir2"), ] for _dir in dirs: ensure_dir(_dir) retcode = wrap_and_call( "cloudinit.cmd.clean", - {"Init": {"side_effect": self.init_class}}, + {"Init": {"side_effect": init_class}}, clean.remove_artifacts, remove_logs=False, ) - self.assertEqual(0, retcode) + assert 0 == retcode for expected_dir in dirs[:2]: - self.assertTrue( - os.path.exists(expected_dir), - "Missing {0} dir".format(expected_dir), - ) + assert expected_dir.exists() is True, f"Missing {expected_dir}" for deleted_dir in dirs[2:]: - self.assertFalse( - os.path.exists(deleted_dir), - "Unexpected {0} dir".format(deleted_dir), - ) + assert deleted_dir.exists() is False, f"Unexpected {deleted_dir}" - def test_remove_artifacts_removes_artifacts_removes_seed(self): + def test_remove_artifacts_removes_artifacts_removes_seed( + self, clean_paths, init_class + ): """remove_artifacts removes seed dir when remove_seed is True.""" dirs = [ - self.artifact_dir, - os.path.join(self.artifact_dir, "seed"), - os.path.join(self.artifact_dir, "dir1"), - os.path.join(self.artifact_dir, "dir2"), + clean_paths.cloud_dir, + clean_paths.cloud_dir.join("seed"), + clean_paths.cloud_dir.join("dir1"), + clean_paths.cloud_dir.join("dir2"), ] for _dir in dirs: ensure_dir(_dir) retcode = wrap_and_call( "cloudinit.cmd.clean", - {"Init": {"side_effect": self.init_class}}, + {"Init": {"side_effect": init_class}}, clean.remove_artifacts, remove_logs=False, remove_seed=True, ) - self.assertEqual(0, retcode) - self.assertTrue( - os.path.exists(self.artifact_dir), "Missing artifact dir" - ) + assert 0 == retcode + assert ( + clean_paths.cloud_dir.exists() is True + ), f"Missing dir {clean_paths.cloud_dir}" for deleted_dir in dirs[1:]: - self.assertFalse( - os.path.exists(deleted_dir), - "Unexpected {0} dir".format(deleted_dir), - ) + assert ( + deleted_dir.exists() is False + ), f"Unexpected {deleted_dir} dir" - def test_remove_artifacts_returns_one_on_errors(self): + def test_remove_artifacts_returns_one_on_errors( + self, clean_paths, init_class, capsys + ): """remove_artifacts returns non-zero on failure and prints an error.""" - ensure_dir(self.artifact_dir) - ensure_dir(os.path.join(self.artifact_dir, "dir1")) + ensure_dir(clean_paths.cloud_dir) + ensure_dir(clean_paths.cloud_dir.join("dir1")) - with mock.patch("sys.stderr", new_callable=StringIO) as m_stderr: - retcode = wrap_and_call( - "cloudinit.cmd.clean", - { - "del_dir": {"side_effect": OSError("oops")}, - "Init": {"side_effect": self.init_class}, - }, - clean.remove_artifacts, - remove_logs=False, - ) - self.assertEqual(1, retcode) - self.assertEqual( - "Error:\nCould not remove %s/dir1: oops\n" % self.artifact_dir, - m_stderr.getvalue(), + retcode = wrap_and_call( + "cloudinit.cmd.clean", + { + "del_dir": {"side_effect": OSError("oops")}, + "Init": {"side_effect": init_class}, + }, + clean.remove_artifacts, + remove_logs=False, + ) + assert 1 == retcode + _out, err = capsys.readouterr() + assert ( + f"Error:\nCould not remove {clean_paths.cloud_dir}/dir1: oops\n" + == err ) - def test_handle_clean_args_reboots(self): + def test_handle_clean_args_reboots(self, init_class): """handle_clean_args_reboots when reboot arg is provided.""" called_cmds = [] @@ -174,38 +219,76 @@ def fake_subp(cmd, capture): called_cmds.append((cmd, capture)) return "", "" - myargs = namedtuple("MyArgs", "remove_logs remove_seed reboot") - cmdargs = myargs(remove_logs=False, remove_seed=False, reboot=True) + myargs = namedtuple( + "MyArgs", "remove_logs remove_seed reboot machine_id" + ) + cmdargs = myargs( + remove_logs=False, remove_seed=False, reboot=True, machine_id=False + ) retcode = wrap_and_call( "cloudinit.cmd.clean", { "subp": {"side_effect": fake_subp}, - "Init": {"side_effect": self.init_class}, + "Init": {"side_effect": init_class}, }, clean.handle_clean_args, name="does not matter", args=cmdargs, ) - self.assertEqual(0, retcode) - self.assertEqual([(["shutdown", "-r", "now"], False)], called_cmds) + assert 0 == retcode + assert [(["shutdown", "-r", "now"], False)] == called_cmds + + @pytest.mark.parametrize("machine_id", (True, False)) + def test_handle_clean_args_removed_machine_id( + self, machine_id, clean_paths, init_class + ): + """handle_clean_args removes /etc/machine-id when arg is True.""" + + myargs = namedtuple( + "MyArgs", "remove_logs remove_seed reboot machine_id" + ) + cmdargs = myargs( + remove_logs=False, + remove_seed=False, + reboot=False, + machine_id=machine_id, + ) + machine_id_path = clean_paths.tmpdir.join("machine-id") + machine_id_path.write("SOME-AMAZN-MACHINE-ID") + with mock.patch.object( + cloudinit.settings, "CLEAN_RUNPARTS_DIR", clean_paths.clean_dir + ): + with mock.patch.object( + cloudinit.cmd.clean, "ETC_MACHINE_ID", machine_id_path.strpath + ): + retcode = wrap_and_call( + "cloudinit.cmd.clean", + { + "Init": {"side_effect": init_class}, + }, + clean.handle_clean_args, + name="does not matter", + args=cmdargs, + ) + assert 0 == retcode + assert machine_id_path.exists() is bool(not machine_id) - def test_status_main(self): + def test_status_main(self, clean_paths, init_class): """clean.main can be run as a standalone script.""" - write_file(self.log1, "cloud-init-log") - with self.assertRaises(SystemExit) as context_manager: + clean_paths.log.write("cloud-init-log") + with pytest.raises(SystemExit) as context_manager: wrap_and_call( "cloudinit.cmd.clean", { - "Init": {"side_effect": self.init_class}, + "Init": {"side_effect": init_class}, "sys.argv": {"new": ["clean", "--logs"]}, }, clean.main, ) - - self.assertEqual(0, context_manager.exception.code) - self.assertFalse( - os.path.exists(self.log1), "Unexpected log {0}".format(self.log1) - ) + assert 0 == context_manager.value.code + assert ( + clean_paths.log.exists() is False + ), f"Unexpected log {clean_paths.log}" # vi: ts=4 expandtab syntax=python diff --git a/tests/unittests/cmd/test_cloud_id.py b/tests/unittests/cmd/test_cloud_id.py index 907297a66..37f1df2cb 100644 --- a/tests/unittests/cmd/test_cloud_id.py +++ b/tests/unittests/cmd/test_cloud_id.py @@ -2,8 +2,6 @@ """Tests for cloud-id command line utility.""" -from collections import namedtuple - import pytest from cloudinit import util @@ -14,9 +12,6 @@ class TestCloudId: - - args = namedtuple("cloudidargs", "instance_data json long") - def test_cloud_id_arg_parser_defaults(self): """Validate the argument defaults when not provided by the end-user.""" cmd = ["cloud-id"] diff --git a/tests/unittests/cmd/test_main.py b/tests/unittests/cmd/test_main.py index 2f7a1fb16..e9ad0bb8c 100644 --- a/tests/unittests/cmd/test_main.py +++ b/tests/unittests/cmd/test_main.py @@ -13,8 +13,7 @@ from cloudinit.util import ensure_dir, load_file, write_file from tests.unittests.helpers import FilesystemMockingTestCase, wrap_and_call -mypaths = namedtuple("MyPaths", "run_dir") -myargs = namedtuple("MyArgs", "debug files force local reporter subcommand") +MyArgs = namedtuple("MyArgs", "debug files force local reporter subcommand") class TestMain(FilesystemMockingTestCase): @@ -58,7 +57,7 @@ def setUp(self): def test_main_init_run_net_runs_modules(self): """Modules like write_files are run in 'net' mode.""" - cmdargs = myargs( + cmdargs = MyArgs( debug=False, files=None, force=False, @@ -104,7 +103,7 @@ def test_main_init_run_net_calls_set_hostname_when_metadata_present(self): } cloud_cfg = safeyaml.dumps(self.cfg) write_file(self.cloud_cfg_file, cloud_cfg) - cmdargs = myargs( + cmdargs = MyArgs( debug=False, files=None, force=False, diff --git a/tests/unittests/cmd/test_query.py b/tests/unittests/cmd/test_query.py index 207078fa0..dd517a4b4 100644 --- a/tests/unittests/cmd/test_query.py +++ b/tests/unittests/cmd/test_query.py @@ -25,7 +25,7 @@ def _gzip_data(data): with BytesIO() as iobuf: - with gzip.GzipFile(mode="wb", fileobj=iobuf) as gzfp: + with gzip.GzipFile(mode="wb", fileobj=iobuf, mtime=0) as gzfp: gzfp.write(data) return iobuf.getvalue() diff --git a/tests/unittests/config/test_apt_configure_sources_list_v1.py b/tests/unittests/config/test_apt_configure_sources_list_v1.py index d4ade106d..52964e10f 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v1.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v1.py @@ -49,7 +49,7 @@ deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted # FIND_SOMETHING_SPECIAL -""" +""" # noqa: E501 class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): diff --git a/tests/unittests/config/test_apt_source_v1.py b/tests/unittests/config/test_apt_source_v1.py index 371963b1e..4ce412ce3 100644 --- a/tests/unittests/config/test_apt_source_v1.py +++ b/tests/unittests/config/test_apt_source_v1.py @@ -16,6 +16,7 @@ from cloudinit import gpg, subp, util from cloudinit.config import cc_apt_configure from tests.unittests.helpers import TestCase +from tests.unittests.util import get_cloud EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 @@ -42,21 +43,6 @@ def update_package_sources(self): return -class FakeDatasource: - """Fake Datasource helper object""" - - def __init__(self): - self.region = "region" - - -class FakeCloud(object): - """Fake Cloud helper object""" - - def __init__(self): - self.distro = FakeDistro() - self.datasource = FakeDatasource() - - class TestAptSourceConfig(TestCase): """TestAptSourceConfig Main Class to test apt_source configs @@ -78,7 +64,7 @@ def setUp(self): self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list" ) - self.fakecloud = FakeCloud() + self.cloud = get_cloud() rpatcher = mock.patch("cloudinit.util.lsb_release") get_rel = rpatcher.start() @@ -125,7 +111,7 @@ def apt_src_basic(self, filename, cfg): """ cfg = self.wrapv1conf(cfg) - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) self.assertTrue(os.path.isfile(filename)) @@ -280,7 +266,7 @@ def apt_src_replacement(self, filename, cfg): """ cfg = self.wrapv1conf(cfg) params = self._get_default_params() - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) self.assertTrue(os.path.isfile(filename)) @@ -371,7 +357,7 @@ def apt_src_keyid(self, filename, cfg, keynum): cfg = self.wrapv1conf(cfg) with mock.patch.object(cc_apt_configure, "add_apt_key") as mockobj: - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) # check if it added the right number of keys calls = [] @@ -497,7 +483,7 @@ def apt_src_key(self, filename, cfg): cfg = self.wrapv1conf([cfg]) with mock.patch.object(cc_apt_configure, "add_apt_key") as mockobj: - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) # check if it added the right amount of keys sources = cfg["apt"]["sources"] @@ -558,7 +544,7 @@ def test_apt_src_keyonly(self): cfg = {"key": "fakekey 4242", "filename": self.aptlistfile} cfg = self.wrapv1conf([cfg]) with mock.patch.object(cc_apt_configure, "apt_key") as mockobj: - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) calls = ( call( @@ -582,9 +568,7 @@ def test_apt_src_keyidonly(self): subp, "subp", return_value=("fakekey 1212", "") ): with mock.patch.object(cc_apt_configure, "apt_key") as mockobj: - cc_apt_configure.handle( - "test", cfg, self.fakecloud, None, None - ) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) calls = ( call( @@ -613,9 +597,7 @@ def apt_src_keyid_real(self, cfg, expectedkey, is_hardened=None): with mock.patch.object( gpg, "getkeybyid", return_value=expectedkey ) as mockgetkey: - cc_apt_configure.handle( - "test", cfg, self.fakecloud, None, None - ) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) if is_hardened is not None: mockkey.assert_called_with( expectedkey, self.aptlistfile, hardened=is_hardened @@ -661,7 +643,7 @@ def test_apt_src_ppa(self): cfg = self.wrapv1conf([cfg]) with mock.patch.object(subp, "subp") as mockobj: - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) mockobj.assert_called_once_with( [ "add-apt-repository", @@ -691,7 +673,7 @@ def test_apt_src_ppa_tri(self): cfg = self.wrapv1conf([cfg1, cfg2, cfg3]) with mock.patch.object(subp, "subp") as mockobj: - cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + cc_apt_configure.handle("test", cfg, self.cloud, None, None) calls = [ call( [ diff --git a/tests/unittests/config/test_apt_source_v3.py b/tests/unittests/config/test_apt_source_v3.py index 8aceff060..5bb873853 100644 --- a/tests/unittests/config/test_apt_source_v3.py +++ b/tests/unittests/config/test_apt_source_v3.py @@ -45,20 +45,6 @@ } -class FakeDatasource: - """Fake Datasource helper object""" - - def __init__(self): - self.region = "region" - - -class FakeCloud: - """Fake Cloud helper object""" - - def __init__(self): - self.datasource = FakeDatasource() - - class TestAptSourceConfig(t_help.FilesystemMockingTestCase): """TestAptSourceConfig Main Class to test apt configs @@ -690,7 +676,7 @@ def test_apt_v3_list_rename(self, m_get_dpkg_architecture): fromfn = "%s/%s_%s" % (pre, archive, post) tofn = "%s/test.ubuntu.com_%s" % (pre, post) - mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(), arch) + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, get_cloud(), arch) self.assertEqual( mirrors["MIRROR"], "http://test.ubuntu.com/%s/" % component @@ -785,7 +771,7 @@ def test_apt_v3_mirror(self): } mirrors = cc_apt_configure.find_apt_mirror_info( - cfg, FakeCloud(), "amd64" + cfg, get_cloud(), "amd64" ) self.assertEqual(mirrors["MIRROR"], pmir) @@ -821,7 +807,7 @@ def test_apt_v3_mirror_arches(self): ], } - mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(), arch) + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, get_cloud(), arch) self.assertEqual(mirrors["PRIMARY"], pmir) self.assertEqual(mirrors["MIRROR"], pmir) @@ -843,7 +829,7 @@ def test_apt_v3_mirror_arches_default(self): } mirrors = cc_apt_configure.find_apt_mirror_info( - cfg, FakeCloud(), "amd64" + cfg, get_cloud(), "amd64" ) self.assertEqual(mirrors["MIRROR"], pmir) @@ -911,7 +897,7 @@ def test_apt_v3_mirror_search(self): side_effect=[pmir, smir], ) as mocksearch: mirrors = cc_apt_configure.find_apt_mirror_info( - cfg, FakeCloud(), "amd64" + cfg, get_cloud(), "amd64" ) calls = [call(["pfailme", pmir]), call(["sfailme", smir])] @@ -961,7 +947,7 @@ def test_apt_v3_mirror_search_many2(self): cc_apt_configure.util, "search_for_mirror" ) as mockse: mirrors = cc_apt_configure.find_apt_mirror_info( - cfg, FakeCloud(), arch + cfg, get_cloud(), arch ) mockse.assert_not_called() diff --git a/tests/unittests/config/test_cc_ansible.py b/tests/unittests/config/test_cc_ansible.py new file mode 100644 index 000000000..6f71add3b --- /dev/null +++ b/tests/unittests/config/test_cc_ansible.py @@ -0,0 +1,362 @@ +import re +from copy import deepcopy +from textwrap import dedent +from unittest import mock +from unittest.mock import call + +from pytest import mark, param, raises + +from cloudinit import util +from cloudinit.config import cc_ansible +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema +from tests.unittests.util import get_cloud + +distro_version = dedent( + """ansible 2.10.8 + config file = None + configured module search path = ['/home/holmanb/.ansible/plugins/modules', \ + '/usr/share/ansible/plugins/modules'] + ansible python module location = /usr/lib/python3/dist-packages/ansible + executable location = /usr/bin/ansible + python version = 3.10.4 (main, Jun 29 2022, 12:14:53) [GCC 11.2.0]""" +) +pip_version = dedent( + """ansible-pull [core 2.13.2] + config file = None + configured module search path = ['/root/.ansible/plugins/modules', \ + '/usr/share/ansible/plugins/modules'] + ansible python module location = /root/.local/lib/python3.8/site-\ + packages/ansible + ansible collection location = /root/.ansible/collections:\ + /usr/share/ansible/collections + executable location = /root/.local/lib/python3.8/site-packages/\ + ansible/__main__.py + python version = 3.8.10 (default, Jun 22 2022, 20:18:18) [GCC 9.4.0] + jinja version = 3.1.2 + libyaml = True """ +) + +CFG_FULL = { + "ansible": { + "install-method": "distro", + "package-name": "ansible-core", + "pull": { + "url": "https://github/holmanb/vmboot", + "playbook-name": "arch.yml", + "accept-host-key": True, + "clean": True, + "full": True, + "diff": False, + "ssh-common-args": "-y", + "scp-extra-args": "-l", + "sftp-extra-args": "-f", + "checkout": "tree", + "module-path": "~/.ansible/plugins/modules:" + "/usr/share/ansible/plugins/modules", + "timeout": "10", + "vault-id": "me", + "connection": "smart", + "vault-password-file": "/path/to/file", + "module-name": "git", + "sleep": "1", + "tags": "cumulus", + "skip-tags": "cisco", + "private-key": "{nope}", + }, + } +} +CFG_MINIMAL = { + "ansible": { + "install-method": "pip", + "package-name": "ansible", + "pull": { + "url": "https://github/holmanb/vmboot", + "playbook-name": "ubuntu.yml", + }, + } +} + + +class TestSetPasswordsSchema: + @mark.parametrize( + ("config", "error_msg"), + ( + param( + CFG_MINIMAL, + None, + id="essentials", + ), + param( + { + "ansible": { + "install-method": "distro", + "pull": { + "url": "https://github/holmanb/vmboot", + "playbook-name": "centos.yml", + "dance": "bossa nova", + }, + } + }, + "Additional properties are not allowed ", + id="additional-properties", + ), + param( + CFG_FULL, + None, + id="all-keys", + ), + param( + { + "ansible": { + "install-method": "true", + "pull": { + "url": "https://github/holmanb/vmboot", + "playbook-name": "debian.yml", + }, + } + }, + "'true' is not one of ['distro', 'pip']", + id="install-type", + ), + param( + { + "ansible": { + "install-method": "pip", + "pull": { + "playbook-name": "fedora.yml", + }, + } + }, + "'url' is a required property", + id="require-url", + ), + param( + { + "ansible": { + "install-method": "pip", + "pull": { + "url": "gophers://encrypted-gophers/", + }, + } + }, + "'playbook-name' is a required property", + id="require-url", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with raises(SchemaValidationError, match=re.escape(error_msg)): + validate_cloudconfig_schema(config, get_schema(), strict=True) + + +class TestAnsible: + def test_filter_args(self): + """only diff should be removed""" + out = cc_ansible.filter_args( + CFG_FULL.get("ansible", {}).get("pull", {}) + ) + assert out == { + "url": "https://github/holmanb/vmboot", + "playbook-name": "arch.yml", + "accept-host-key": True, + "clean": True, + "full": True, + "ssh-common-args": "-y", + "scp-extra-args": "-l", + "sftp-extra-args": "-f", + "checkout": "tree", + "module-path": "~/.ansible/plugins/modules:" + "/usr/share/ansible/plugins/modules", + "timeout": "10", + "vault-id": "me", + "connection": "smart", + "vault-password-file": "/path/to/file", + "module-name": "git", + "sleep": "1", + "tags": "cumulus", + "skip-tags": "cisco", + "private-key": "{nope}", + } + + @mark.parametrize( + ("cfg", "exception"), + ( + (CFG_FULL, None), + (CFG_MINIMAL, None), + ( + { + "ansible": { + "package-name": "ansible-core", + "install-method": "distro", + "pull": { + "playbook-name": "ubuntu.yml", + }, + } + }, + ValueError, + ), + ( + { + "ansible": { + "install-method": "pip", + "pull": { + "url": "https://github/holmanb/vmboot", + }, + } + }, + ValueError, + ), + ), + ) + def test_required_keys(self, cfg, exception, mocker): + m_subp = mocker.patch( + "cloudinit.config.cc_ansible.subp", return_value=("", "") + ) + mocker.patch("cloudinit.config.cc_ansible.which", return_value=True) + mocker.patch( + "cloudinit.config.cc_ansible.AnsiblePull.get_version", + return_value=cc_ansible.Version(2, 7, 1), + ) + mocker.patch("cloudinit.config.cc_ansible.AnsiblePull.check_deps") + mocker.patch( + "cloudinit.config.cc_ansible.AnsiblePullDistro.is_installed", + return_value=False, + ) + if exception: + with raises(exception): + cc_ansible.handle("", cfg, get_cloud(), None, None) + else: + cloud = get_cloud(mocked_distro=True) + print(cfg) + install = cfg["ansible"]["install-method"] + cc_ansible.handle("", cfg, cloud, None, None) + if install == "distro": + cloud.distro.install_packages.assert_called_once() + cloud.distro.install_packages.assert_called_with( + "ansible-core" + ) + elif install == "pip": + m_subp.assert_has_calls( + [ + call(["python3", "-m", "pip", "list"]), + call( + [ + "python3", + "-m", + "pip", + "install", + "--user", + "ansible", + ] + ), + ] + ) + assert m_subp.call_args[0][0] == [ + "ansible-pull", + "--url=https://github/holmanb/vmboot", + "ubuntu.yml", + ] + + @mock.patch("cloudinit.config.cc_ansible.which", return_value=False) + def test_deps_not_installed(self, m_which): + with raises(ValueError): + cc_ansible.AnsiblePullDistro(get_cloud().distro).check_deps() + + @mock.patch("cloudinit.config.cc_ansible.which", return_value=True) + def test_deps(self, m_which): + cc_ansible.AnsiblePullDistro(get_cloud().distro).check_deps() + + @mock.patch("cloudinit.config.cc_ansible.which", return_value=True) + @mock.patch( + "cloudinit.config.cc_ansible.subp", return_value=("stdout", "stderr") + ) + @mark.parametrize( + ("cfg", "expected"), + ( + ( + CFG_FULL, + [ + "ansible-pull", + "--url=https://github/holmanb/vmboot", + "--accept-host-key", + "--clean", + "--full", + "--ssh-common-args=-y", + "--scp-extra-args=-l", + "--sftp-extra-args=-f", + "--checkout=tree", + "--module-path=~/.ansible/plugins/modules" + ":/usr/share/ansible/plugins/modules", + "--timeout=10", + "--vault-id=me", + "--connection=smart", + "--vault-password-file=/path/to/file", + "--module-name=git", + "--sleep=1", + "--tags=cumulus", + "--skip-tags=cisco", + "--private-key={nope}", + "arch.yml", + ], + ), + ( + CFG_MINIMAL, + [ + "ansible-pull", + "--url=https://github/holmanb/vmboot", + "ubuntu.yml", + ], + ), + ), + ) + def test_ansible_pull(self, m_subp, m_which, cfg, expected): + pull_type = cfg["ansible"]["install-method"] + ansible_pull = ( + cc_ansible.AnsiblePullPip() + if pull_type == "pip" + else cc_ansible.AnsiblePullDistro(get_cloud().distro) + ) + cc_ansible.run_ansible_pull( + ansible_pull, deepcopy(cfg["ansible"]["pull"]) + ) + assert m_subp.call_args[0][0] == expected + + @mock.patch("cloudinit.config.cc_ansible.validate_config") + def test_do_not_run(self, m_validate): + cc_ansible.handle("", {}, None, None, None) # pyright: ignore + assert not m_validate.called + + @mock.patch( + "cloudinit.config.cc_ansible.subp", + side_effect=[ + (distro_version, ""), + (pip_version, ""), + (" ansible 2.1.0", ""), + (" ansible 2.1.0", ""), + ], + ) + def test_parse_version(self, m_subp): + assert cc_ansible.AnsiblePullDistro( + get_cloud().distro + ).get_version() == cc_ansible.Version(2, 10, 8) + assert cc_ansible.AnsiblePullPip().get_version() == cc_ansible.Version( + 2, 13, 2 + ) + + assert ( + util.Version(2, 1, 0, -1) + == cc_ansible.AnsiblePullPip().get_version() + ) + assert ( + util.Version(2, 1, 0, -1) + == cc_ansible.AnsiblePullDistro(get_cloud().distro).get_version() + ) diff --git a/tests/unittests/config/test_cc_ca_certs.py b/tests/unittests/config/test_cc_ca_certs.py index 396146350..a0b402acd 100644 --- a/tests/unittests/config/test_cc_ca_certs.py +++ b/tests/unittests/config/test_cc_ca_certs.py @@ -421,7 +421,14 @@ class TestCACertsSchema: "config, error_msg", ( # Valid, yet deprecated schemas - ({"ca-certs": {"remove-defaults": True}}, None), + ( + {"ca-certs": {"remove-defaults": True}}, + "Cloud config schema deprecations: " + "ca-certs: DEPRECATED. Dropped after April 2027. " + "Use ``ca_certs``., " + "ca-certs.remove-defaults: DEPRECATED. " + "Dropped after April 2027. Use ``remove_defaults``.", + ), # Invalid schemas ( {"ca_certs": 1}, diff --git a/tests/unittests/config/test_cc_debug.py b/tests/unittests/config/test_cc_debug.py deleted file mode 100644 index fc8d43dc2..000000000 --- a/tests/unittests/config/test_cc_debug.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (C) 2014 Yahoo! Inc. -# -# This file is part of cloud-init. See LICENSE file for license information. -import logging -import re -import shutil -import tempfile - -import pytest - -from cloudinit import util -from cloudinit.config import cc_debug -from cloudinit.config.schema import ( - SchemaValidationError, - get_schema, - validate_cloudconfig_schema, -) -from tests.unittests.helpers import ( - FilesystemMockingTestCase, - mock, - skipUnlessJsonSchema, -) -from tests.unittests.util import get_cloud - -LOG = logging.getLogger(__name__) - - -@mock.patch("cloudinit.distros.debian.read_system_locale") -class TestDebug(FilesystemMockingTestCase): - def setUp(self): - super(TestDebug, self).setUp() - self.new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.new_root) - self.patchUtils(self.new_root) - - def test_debug_write(self, m_locale): - m_locale.return_value = "en_US.UTF-8" - cfg = { - "abc": "123", - "c": "\u20a0", - "debug": { - "verbose": True, - # Does not actually write here due to mocking... - "output": "/var/log/cloud-init-debug.log", - }, - } - cc = get_cloud() - cc_debug.handle("cc_debug", cfg, cc, LOG, []) - contents = util.load_file("/var/log/cloud-init-debug.log") - # Some basic sanity tests... - self.assertNotEqual(0, len(contents)) - for k in cfg.keys(): - self.assertIn(k, contents) - - def test_debug_no_write(self, m_locale): - m_locale.return_value = "en_US.UTF-8" - cfg = { - "abc": "123", - "debug": { - "verbose": False, - # Does not actually write here due to mocking... - "output": "/var/log/cloud-init-debug.log", - }, - } - cc = get_cloud() - cc_debug.handle("cc_debug", cfg, cc, LOG, []) - self.assertRaises( - IOError, util.load_file, "/var/log/cloud-init-debug.log" - ) - - -@skipUnlessJsonSchema() -class TestDebugSchema: - """Directly test schema rather than through handle.""" - - @pytest.mark.parametrize( - "config, error_msg", - ( - # Valid schemas tested by meta.examples in test_schema - # Invalid schemas - ({"debug": 1}, "debug: 1 is not of type 'object'"), - ( - {"debug": {}}, - re.escape("debug: {} does not have enough properties"), - ), - ( - {"debug": {"boguskey": True}}, - re.escape( - "Additional properties are not allowed ('boguskey' was" - " unexpected)" - ), - ), - ( - {"debug": {"verbose": 1}}, - "debug.verbose: 1 is not of type 'boolean'", - ), - ( - {"debug": {"output": 1}}, - "debug.output: 1 is not of type 'string'", - ), - ), - ) - @skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): - """Assert expected schema validation and error messages.""" - # New-style schema $defs exist in config/cloud-init-schema*.json - schema = get_schema() - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, schema, strict=True) - - -# vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py index f2796e839..c61a26f33 100644 --- a/tests/unittests/config/test_cc_disk_setup.py +++ b/tests/unittests/config/test_cc_disk_setup.py @@ -1,7 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import random -import re import pytest @@ -312,13 +311,6 @@ class TestDebugSchema: {"device_aliases": 1}, "device_aliases: 1 is not of type 'object'", ), - ( - {"debug": {"boguskey": True}}, - re.escape( - "Additional properties are not allowed ('boguskey' was" - " unexpected)" - ), - ), ), ) @skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_cc_growpart.py b/tests/unittests/config/test_cc_growpart.py index 24e92c88e..f4d4e579a 100644 --- a/tests/unittests/config/test_cc_growpart.py +++ b/tests/unittests/config/test_cc_growpart.py @@ -20,7 +20,11 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import TestCase, skipUnlessJsonSchema +from tests.unittests.helpers import ( + TestCase, + does_not_raise, + skipUnlessJsonSchema, +) # growpart: # mode: auto # off, on, auto, 'growpart' @@ -591,37 +595,65 @@ def __init__(self, **kwds): class TestGrowpartSchema: @pytest.mark.parametrize( - "config, error_msg", + "config, expectation", ( - ({"growpart": {"mode": "off"}}, None), - ({"growpart": {"mode": False}}, None), + ({"growpart": {"mode": "off"}}, does_not_raise()), + ( + {"growpart": {"mode": False}}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: growpart.mode: DEPRECATED. Specifying" + " a boolean ``false`` value for this key is" + " deprecated. Use ``off`` instead." + ), + ), + ), ( {"growpart": {"mode": "false"}}, - "'false' is not one of " - r"\[False, 'auto', 'growpart', 'gpart', 'off'\]", + pytest.raises( + SchemaValidationError, + match=( + "growpart.mode: 'false' is not valid under any of the" + " given schemas" + ), + ), ), ( {"growpart": {"mode": "a"}}, - "'a' is not one of " - r"\[False, 'auto', 'growpart', 'gpart', 'off'\]", + pytest.raises( + SchemaValidationError, + match=( + "growpart.mode: 'a' is not valid under any of the" + " given schemas" + ), + ), + ), + ( + {"growpart": {"devices": "/"}}, + pytest.raises( + SchemaValidationError, match="'/' is not of type 'array'" + ), ), - ({"growpart": {"devices": "/"}}, "'/' is not of type 'array'"), ( {"growpart": {"ignore_growroot_disabled": "off"}}, - "'off' is not of type 'boolean'", + pytest.raises( + SchemaValidationError, + match="'off' is not of type 'boolean'", + ), ), ( {"growpart": {"a": "b"}}, - "Additional properties are not allowed", + pytest.raises( + SchemaValidationError, + match="Additional properties are not allowed", + ), ), ), ) @skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): + def test_schema_validation(self, config, expectation): """Assert expected schema validation and error messages.""" schema = get_schema() - if error_msg is None: + with expectation: validate_cloudconfig_schema(config, schema, strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, schema, strict=True) diff --git a/tests/unittests/config/test_cc_grub_dpkg.py b/tests/unittests/config/test_cc_grub_dpkg.py index 9bdc9c742..0f9cc2321 100644 --- a/tests/unittests/config/test_cc_grub_dpkg.py +++ b/tests/unittests/config/test_cc_grub_dpkg.py @@ -12,7 +12,7 @@ validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import skipUnlessJsonSchema +from tests.unittests.helpers import does_not_raise, skipUnlessJsonSchema class TestFetchIdevs: @@ -194,26 +194,59 @@ def test_handle( class TestGrubDpkgSchema: @pytest.mark.parametrize( - "config, error_msg", + "config, expectation, has_errors", ( - ({"grub_dpkg": {"grub-pc/install_devices_empty": False}}, None), - ({"grub_dpkg": {"grub-pc/install_devices_empty": "off"}}, None), + ( + {"grub_dpkg": {"grub-pc/install_devices_empty": False}}, + does_not_raise(), + None, + ), + ( + {"grub_dpkg": {"grub-pc/install_devices_empty": "off"}}, + pytest.raises( + SchemaValidationError, + match=( + r"^Cloud config schema deprecations:" + r" grub_dpkg.grub-pc/install_devices_empty:" + r" DEPRECATED. Use a boolean value instead.$" + ), + ), + False, + ), ( {"grub_dpkg": {"enabled": "yes"}}, - "'yes' is not of type 'boolean'", + pytest.raises( + SchemaValidationError, + match="'yes' is not of type 'boolean'", + ), + True, ), ( {"grub_dpkg": {"grub-pc/install_devices": ["/dev/sda"]}}, - r"\['/dev/sda'\] is not of type 'string'", + pytest.raises( + SchemaValidationError, + match=r"\['/dev/sda'\] is not of type 'string'", + ), + True, + ), + ( + {"grub-dpkg": {"grub-pc/install_devices_empty": False}}, + pytest.raises( + SchemaValidationError, + match=( + r"^Cloud config schema deprecations: grub-dpkg:" + r" DEPRECATED. Use ``grub_dpkg`` instead$" + ), + ), + False, ), ), ) @skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): + def test_schema_validation(self, config, expectation, has_errors): """Assert expected schema validation and error messages.""" schema = get_schema() - if error_msg is None: + with expectation as exc_info: validate_cloudconfig_schema(config, schema, strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, schema, strict=True) + if has_errors is not None: + assert has_errors == exc_info.value.has_errors() diff --git a/tests/unittests/config/test_cc_landscape.py b/tests/unittests/config/test_cc_landscape.py index 79ea6b0a7..b08e3d442 100644 --- a/tests/unittests/config/test_cc_landscape.py +++ b/tests/unittests/config/test_cc_landscape.py @@ -186,6 +186,15 @@ class TestLandscapeSchema: # tags are comma-delimited ({"landscape": {"client": {"tags": "1,2,3"}}}, None), ({"landscape": {"client": {"tags": "1"}}}, None), + ( + { + "landscape": { + "client": {}, + "random-config-value": {"tags": "1"}, + } + }, + "Additional properties are not allowed", + ), # Require client key ({"landscape": {}}, "'client' is a required property"), # tags are not whitespace-delimited diff --git a/tests/unittests/config/test_cc_lxd.py b/tests/unittests/config/test_cc_lxd.py index 3b444127b..8b75a1f7a 100644 --- a/tests/unittests/config/test_cc_lxd.py +++ b/tests/unittests/config/test_cc_lxd.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import re +from copy import deepcopy from unittest import mock import pytest @@ -27,33 +28,82 @@ class TestLxd(t_help.CiTestCase): } } } + backend_def = ( + ("zfs", "zfs", "zfsutils-linux"), + ("btrfs", "mkfs.btrfs", "btrfs-progs"), + ("lvm", "lvcreate", "lvm2"), + ("dir", None, None), + ) - @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.subp") - def test_lxd_init(self, mock_subp, m_maybe_clean): - cc = get_cloud() - mock_subp.which.return_value = True - m_maybe_clean.return_value = None - cc_lxd.handle("cc_lxd", self.lxd_cfg, cc, self.logger, []) - self.assertTrue(mock_subp.which.called) - # no bridge config, so maybe_cleanup should not be called. - self.assertFalse(m_maybe_clean.called) - self.assertEqual( - [ - mock.call(["lxd", "waitready", "--timeout=300"]), - mock.call( + @mock.patch("cloudinit.config.cc_lxd.util.system_info") + @mock.patch("cloudinit.config.cc_lxd.os.path.exists", return_value=True) + @mock.patch("cloudinit.config.cc_lxd.subp.subp", return_value=True) + @mock.patch("cloudinit.config.cc_lxd.subp.which", return_value=False) + @mock.patch( + "cloudinit.config.cc_lxd.maybe_cleanup_default", return_value=None + ) + def test_lxd_init(self, maybe_clean, which, subp, exists, system_info): + system_info.return_value = {"uname": [0, 1, "mykernel"]} + cc = get_cloud(mocked_distro=True) + install = cc.distro.install_packages + + for backend, cmd, package in self.backend_def: + lxd_cfg = deepcopy(self.lxd_cfg) + lxd_cfg["lxd"]["init"]["storage_backend"] = backend + subp.call_args_list = [] + install.call_args_list = [] + exists.call_args_list = [] + cc_lxd.handle("cc_lxd", lxd_cfg, cc, self.logger, []) + if cmd: + which.assert_called_with(cmd) + # no bridge config, so maybe_cleanup should not be called. + self.assertFalse(maybe_clean.called) + self.assertEqual( + [ + mock.call(list(filter(None, ["lxd", package]))), + ], + install.call_args_list, + ) + self.assertEqual( + [ + mock.call(["lxd", "waitready", "--timeout=300"]), + mock.call( + [ + "lxd", + "init", + "--auto", + "--network-address=0.0.0.0", + f"--storage-backend={backend}", + "--storage-pool=poolname", + ] + ), + ], + subp.call_args_list, + ) + + if backend == "lvm": + self.assertEqual( [ - "lxd", - "init", - "--auto", - "--network-address=0.0.0.0", - "--storage-backend=zfs", - "--storage-pool=poolname", - ] - ), - ], - mock_subp.subp.call_args_list, - ) + mock.call( + "/lib/modules/mykernel/" + "kernel/drivers/md/dm-thin-pool.ko" + ) + ], + exists.call_args_list, + ) + else: + self.assertEqual([], exists.call_args_list) + + @mock.patch("cloudinit.config.cc_lxd.subp.which", return_value=False) + def test_lxd_package_install(self, m_which): + for backend, _, package in self.backend_def: + lxd_cfg = deepcopy(self.lxd_cfg) + lxd_cfg["lxd"]["init"]["storage_backend"] = backend + + packages = cc_lxd.get_required_packages(lxd_cfg["lxd"]["init"]) + assert "lxd" in packages + if package: + assert package in packages @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") @mock.patch("cloudinit.config.cc_lxd.subp") @@ -174,6 +224,7 @@ def test_lxd_cmd_new_full(self): "ipv6_netmask": "64", "ipv6_nat": "true", "domain": "lxd", + "mtu": 9000, } self.assertEqual( cc_lxd.bridge_to_cmd(data), @@ -188,6 +239,7 @@ def test_lxd_cmd_new_full(self): "ipv6.address=fd98:9e0:3744::1/64", "ipv6.nat=true", "dns.domain=lxd", + "bridge.mtu=9000", ], ["network", "attach-profile", "testbr0", "default", "eth0"], ), @@ -199,6 +251,7 @@ def test_lxd_cmd_new_partial(self): "ipv6_address": "fd98:9e0:3744::1", "ipv6_netmask": "64", "ipv6_nat": "true", + "mtu": -1, } self.assertEqual( cc_lxd.bridge_to_cmd(data), @@ -286,17 +339,36 @@ class TestLXDSchema: # Only allow init.storage_backend values zfs and dir ( {"lxd": {"init": {"storage_backend": "1zfs"}}}, - re.escape("not one of ['zfs', 'dir']"), + re.escape("not one of ['zfs', 'dir', 'lvm', 'btrfs']"), ), + ({"lxd": {"init": {"storage_backend": "lvm"}}}, None), + ({"lxd": {"init": {"storage_backend": "btrfs"}}}, None), + ({"lxd": {"init": {"storage_backend": "zfs"}}}, None), # Require bridge.mode ({"lxd": {"bridge": {}}}, "bridge: 'mode' is a required property"), # Require init or bridge keys ({"lxd": {}}, "does not have enough properties"), + # Require bridge.mode + ({"lxd": {"bridge": {"mode": "new", "mtu": 9000}}}, None), + # LXD's default value + ({"lxd": {"bridge": {"mode": "new", "mtu": -1}}}, None), + # No additionalProperties + ( + {"lxd": {"init": {"invalid": None}}}, + "Additional properties are not allowed", + ), + ( + {"lxd": {"bridge": {"mode": None, "garbage": None}}}, + "Additional properties are not allowed", + ), ], ) @t_help.skipUnlessJsonSchema() def test_schema_validation(self, config, error_msg): - with pytest.raises(SchemaValidationError, match=error_msg): + if error_msg: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_mcollective.py b/tests/unittests/config/test_cc_mcollective.py index aa726dd39..a581f9bb7 100644 --- a/tests/unittests/config/test_cc_mcollective.py +++ b/tests/unittests/config/test_cc_mcollective.py @@ -172,6 +172,11 @@ class TestMcollectiveSchema: ), # Allow undocumented keys client keys below 'conf' without error ({"mcollective": {"conf": {"customkey": 1}}}, None), + # Don't allow undocumented keys that don't match expected type + ( + {"mcollective": {"conf": {"": {"test": None}}}}, + "does not match any of the regexes:", + ), ( {"mcollective": {"conf": {"public-cert": 1}}}, "mcollective.conf.public-cert: 1 is not of type 'string'", diff --git a/tests/unittests/config/test_cc_mounts.py b/tests/unittests/config/test_cc_mounts.py index 8ae280994..0073829af 100644 --- a/tests/unittests/config/test_cc_mounts.py +++ b/tests/unittests/config/test_cc_mounts.py @@ -1,13 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. +import math import os.path import re +from collections import namedtuple from unittest import mock import pytest +from pytest import approx from cloudinit.config import cc_mounts -from cloudinit.config.cc_mounts import create_swapfile +from cloudinit.config.cc_mounts import ( + GB, + MB, + create_swapfile, + suggested_swapsize, +) from cloudinit.config.schema import ( SchemaValidationError, get_schema, @@ -524,6 +532,35 @@ def subp_side_effect(cmd, *args, **kwargs): msg = "fallocate swap creation failed, will attempt with dd" assert msg in caplog.text + # See https://help.ubuntu.com/community/SwapFaq + @pytest.mark.parametrize( + "memsize,expected", + [ + (256 * MB, 256 * MB), + (512 * MB, 512 * MB), + (1 * GB, 1 * GB), + (2 * GB, 2 * GB), + (4 * GB, 4 * GB), + (8 * GB, 4 * GB), + (16 * GB, 4 * GB), + (32 * GB, 6 * GB), + (64 * GB, 8 * GB), + (128 * GB, 11 * GB), + (256 * GB, 16 * GB), + (512 * GB, 23 * GB), + ], + ) + def test_suggested_swapsize(self, memsize, expected, mocker): + mock_stat = namedtuple("mock_stat", "f_frsize f_bfree") + mocker.patch( + "os.statvfs", + # Don't care about available disk space for the purposes of this + # test + return_value=mock_stat(math.inf, math.inf), + ) + size = suggested_swapsize(memsize, math.inf, "dontcare") + assert expected == approx(size) + class TestMountsSchema: @pytest.mark.parametrize( diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py index c2bce2a3f..41b5fb9b5 100644 --- a/tests/unittests/config/test_cc_ntp.py +++ b/tests/unittests/config/test_cc_ntp.py @@ -499,15 +499,6 @@ def test_opensuse_picks_chrony(self, m_sysinfo): expected_client = mycloud.distro.preferred_ntp_clients[0] self.assertEqual("chrony", expected_client) - @mock.patch("cloudinit.util.system_info") - def test_ubuntu_xenial_picks_ntp(self, m_sysinfo): - """Test Ubuntu picks ntp on xenial release""" - - m_sysinfo.return_value = {"dist": ("Ubuntu", "16.04", "xenial")} - mycloud = self._get_cloud("ubuntu") - expected_client = mycloud.distro.preferred_ntp_clients[0] - self.assertEqual("ntp", expected_client) - @mock.patch("cloudinit.config.cc_ntp.subp.which") def test_snappy_system_picks_timesyncd(self, m_which): """Test snappy systems prefer installed clients""" diff --git a/tests/unittests/config/test_cc_package_update_upgrade_install.py b/tests/unittests/config/test_cc_package_update_upgrade_install.py index 1bdddfcc1..e8fce98fc 100644 --- a/tests/unittests/config/test_cc_package_update_upgrade_install.py +++ b/tests/unittests/config/test_cc_package_update_upgrade_install.py @@ -18,6 +18,30 @@ class TestPackageUpdateUpgradeSchema: ({"packages": ["p1", ["p2", "p3", "p4"]]}, ""), # empty packages list ({"packages": []}, "is too short"), + ( + {"apt_update": False}, + ( + "deprecations: apt_update: DEPRECATED." + " Dropped after April 2027. Use ``package_update``." + " Default: ``false``" + ), + ), + ( + {"apt_upgrade": False}, + ( + "deprecations: apt_upgrade: DEPRECATED." + " Dropped after April 2027. Use ``package_upgrade``." + " Default: ``false``" + ), + ), + ( + {"apt_reboot_if_required": False}, + ( + "deprecations: apt_reboot_if_required: DEPRECATED." + " Dropped after April 2027." + " Use ``package_reboot_if_required``. Default: ``false``" + ), + ), ], ) @skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_cc_power_state_change.py b/tests/unittests/config/test_cc_power_state_change.py index cdd36fe00..82824306b 100644 --- a/tests/unittests/config/test_cc_power_state_change.py +++ b/tests/unittests/config/test_cc_power_state_change.py @@ -173,9 +173,23 @@ class TestPowerStateChangeSchema: r"'test' is not one of \['poweroff', 'reboot', 'halt'\]", ), # Delay can be a number, a +number, or "now" - ({"power_state": {"mode": "halt", "delay": "5"}}, None), + ( + {"power_state": {"mode": "halt", "delay": "5"}}, + ( + "power_state.delay: DEPRECATED:" + " Use of string for this value will be dropped after" + " April 2027. Use ``now`` or integer type." + ), + ), ({"power_state": {"mode": "halt", "delay": "now"}}, None), - ({"power_state": {"mode": "halt", "delay": "+5"}}, None), + ( + {"power_state": {"mode": "halt", "delay": "+5"}}, + ( + "power_state.delay: DEPRECATED:" + " Use of string for this value will be dropped after" + " April 2027. Use ``now`` or integer type." + ), + ), ({"power_state": {"mode": "halt", "delay": "+"}}, ""), ({"power_state": {"mode": "halt", "delay": "++5"}}, ""), ({"power_state": {"mode": "halt", "delay": "-5"}}, ""), diff --git a/tests/unittests/config/test_cc_rh_subscription.py b/tests/unittests/config/test_cc_rh_subscription.py index 57313361d..1a9c15796 100644 --- a/tests/unittests/config/test_cc_rh_subscription.py +++ b/tests/unittests/config/test_cc_rh_subscription.py @@ -149,7 +149,7 @@ class TestBadInput(CiTestCase): name = "cc_rh_subscription" cloud_init = None log = logging.getLogger("bad_tests") - args = [] + args: list = [] SM = cc_rh_subscription.SubscriptionManager reg = ( "The system has been registered with ID:" diff --git a/tests/unittests/config/test_cc_scripts_vendor.py b/tests/unittests/config/test_cc_scripts_vendor.py index a8cbfb4f4..1dcd0573b 100644 --- a/tests/unittests/config/test_cc_scripts_vendor.py +++ b/tests/unittests/config/test_cc_scripts_vendor.py @@ -5,24 +5,32 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import skipUnlessJsonSchema +from tests.unittests.helpers import does_not_raise, skipUnlessJsonSchema class TestScriptsVendorSchema: @pytest.mark.parametrize( - "config, error_msg", + "config, expectation", ( - ({"vendor_data": {"enabled": True}}, None), - ({"vendor_data": {"enabled": "yes"}}, None), + ({"vendor_data": {"enabled": True}}, does_not_raise()), + ({"vendor_data": {"enabled": False}}, does_not_raise()), + ( + {"vendor_data": {"enabled": "yes"}}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: vendor_data.enabled: DEPRECATED." + " Use of string for this value is DEPRECATED." + " Use a boolean value instead." + ), + ), + ), ), ) @skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): + def test_schema_validation(self, config, expectation): """Assert expected schema validation and error messages.""" # New-style schema $defs exist in config/cloud-init-schema*.json schema = get_schema() - if error_msg is None: + with expectation: validate_cloudconfig_schema(config, schema, strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, schema, strict=True) diff --git a/tests/unittests/config/test_cc_set_hostname.py b/tests/unittests/config/test_cc_set_hostname.py index fd994c4ec..3d1d86eef 100644 --- a/tests/unittests/config/test_cc_set_hostname.py +++ b/tests/unittests/config/test_cc_set_hostname.py @@ -11,6 +11,7 @@ from cloudinit import cloud, distros, helpers, util from cloudinit.config import cc_set_hostname +from cloudinit.sources import DataSourceNone from tests.unittests import helpers as t_help LOG = logging.getLogger(__name__) @@ -153,7 +154,8 @@ def test_photon_hostname(self, m_subp): ) ] not in m_subp.call_args_list - def test_multiple_calls_skips_unchanged_hostname(self): + @mock.patch("cloudinit.util.get_hostname", return_value="localhost") + def test_multiple_calls_skips_unchanged_hostname(self, get_hostname): """Only new hostname or fqdn values will generate a hostname call.""" distro = self._fetch_distro("debian") paths = helpers.Paths({"cloud_dir": self.tmp}) @@ -182,6 +184,42 @@ def test_multiple_calls_skips_unchanged_hostname(self): self.logs.getvalue(), ) + @mock.patch("cloudinit.util.get_hostname", return_value="localhost") + def test_localhost_default_hostname(self, get_hostname): + """ + No hostname set. Default value returned is localhost, + but we shouldn't write it in /etc/hostname + """ + distro = self._fetch_distro("debian") + paths = helpers.Paths({"cloud_dir": self.tmp}) + ds = DataSourceNone.DataSourceNone({}, None, paths) + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + + util.write_file("/etc/hostname", "") + cc_set_hostname.handle("cc_set_hostname", {}, cc, LOG, []) + contents = util.load_file("/etc/hostname") + self.assertEqual("", contents.strip()) + + @mock.patch("cloudinit.util.get_hostname", return_value="localhost") + def test_localhost_user_given_hostname(self, get_hostname): + """ + User set hostname is localhost. We should write it in /etc/hostname + """ + distro = self._fetch_distro("debian") + paths = helpers.Paths({"cloud_dir": self.tmp}) + ds = DataSourceNone.DataSourceNone({}, None, paths) + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + + # user-provided localhost should not be ignored + util.write_file("/etc/hostname", "") + cc_set_hostname.handle( + "cc_set_hostname", {"hostname": "localhost"}, cc, LOG, [] + ) + contents = util.load_file("/etc/hostname") + self.assertEqual("localhost", contents.strip()) + def test_error_on_distro_set_hostname_errors(self): """Raise SetHostnameError on exceptions from distro.set_hostname.""" distro = self._fetch_distro("debian") diff --git a/tests/unittests/config/test_cc_set_passwords.py b/tests/unittests/config/test_cc_set_passwords.py index ac7abadb4..8bf4c36c3 100644 --- a/tests/unittests/config/test_cc_set_passwords.py +++ b/tests/unittests/config/test_cc_set_passwords.py @@ -5,22 +5,24 @@ import pytest -from cloudinit import subp, util +from cloudinit import features, subp, util from cloudinit.config import cc_set_passwords as setpass from cloudinit.config.schema import ( SchemaValidationError, get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import CiTestCase, skipUnlessJsonSchema +from tests.unittests.helpers import does_not_raise, skipUnlessJsonSchema from tests.unittests.util import get_cloud MODPATH = "cloudinit.config.cc_set_passwords." +LOG = logging.getLogger(__name__) -@pytest.fixture() -def mock_uses_systemd(mocker): +@pytest.fixture(autouse=True) +def common_fixtures(mocker): mocker.patch("cloudinit.distros.uses_systemd", return_value=True) + mocker.patch("cloudinit.util.write_to_console") class TestHandleSSHPwauth: @@ -72,7 +74,7 @@ def test_unknown_value_logs_warning( ), ), ) - @mock.patch(MODPATH + "update_ssh_config") + @mock.patch(f"{MODPATH}update_ssh_config") @mock.patch("cloudinit.distros.subp.subp") def test_restart_ssh_only_when_changes_made_and_ssh_installed( self, @@ -98,11 +100,9 @@ def test_restart_ssh_only_when_changes_made_and_ssh_installed( r.msg for r in caplog.records if r.levelname == "DEBUG" ) - @mock.patch(MODPATH + "update_ssh_config", return_value=True) + @mock.patch(f"{MODPATH}update_ssh_config", return_value=True) @mock.patch("cloudinit.distros.subp.subp") - def test_unchanged_value_does_nothing( - self, m_subp, update_ssh_config, mock_uses_systemd - ): + def test_unchanged_value_does_nothing(self, m_subp, update_ssh_config): """If 'unchanged', then no updates to config and no restart.""" update_ssh_config.assert_not_called() cloud = get_cloud("ubuntu") @@ -113,10 +113,10 @@ def test_unchanged_value_does_nothing( @pytest.mark.allow_subp_for("systemctl") @mock.patch("cloudinit.distros.subp.subp") - def test_valid_value_changes_updates_ssh(self, m_subp, mock_uses_systemd): + def test_valid_value_changes_updates_ssh(self, m_subp): """If value is a valid changed value, then update will be called.""" cloud = get_cloud("ubuntu") - upname = MODPATH + "update_ssh_config" + upname = f"{MODPATH}update_ssh_config" optname = "PasswordAuthentication" for n, value in enumerate(util.FALSE_STRINGS + util.TRUE_STRINGS, 1): optval = "yes" if value in util.TRUE_STRINGS else "no" @@ -213,7 +213,7 @@ def test_valid_value_changes_updates_ssh(self, m_subp, mock_uses_systemd): ), ), ) - @mock.patch(MODPATH + "update_ssh_config", return_value=True) + @mock.patch(f"{MODPATH}update_ssh_config", return_value=True) @mock.patch("cloudinit.distros.subp.subp") def test_no_restart_when_service_is_not_running( self, @@ -249,33 +249,42 @@ def test_no_restart_when_service_is_not_running( assert cloud.distro.uses_systemd.call_count == 1 -@pytest.mark.usefixtures("mock_uses_systemd") -class TestSetPasswordsHandle(CiTestCase): - """Test cc_set_passwords.handle""" +def get_chpasswd_calls(cfg, cloud, log): + with mock.patch(f"{MODPATH}subp.subp") as subp: + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle( + "IGNORED", + cfg=cfg, + cloud=cloud, + log=log, + args=[], + ) + assert chpasswd.call_count > 0 + return chpasswd.call_args[0], subp.call_args - with_logs = True - @mock.patch(MODPATH + "subp.subp") - def test_handle_on_empty_config(self, m_subp): +class TestSetPasswordsHandle: + """Test cc_set_passwords.handle""" + + @mock.patch(f"{MODPATH}subp.subp") + def test_handle_on_empty_config(self, m_subp, caplog): """handle logs that no password has changed when config is empty.""" - cloud = self.tmp_cloud(distro="ubuntu") - setpass.handle( - "IGNORED", cfg={}, cloud=cloud, log=self.logger, args=[] - ) - self.assertEqual( - "DEBUG: Leaving SSH config 'PasswordAuthentication' unchanged. " - "ssh_pwauth=None\n", - self.logs.getvalue(), - ) - self.assertEqual( - [mock.call(["systemctl", "status", "ssh"], capture=True)], - m_subp.call_args_list, - ) + cloud = get_cloud() + setpass.handle("IGNORED", cfg={}, cloud=cloud, log=LOG, args=[]) + assert ( + "Leaving SSH config 'PasswordAuthentication' unchanged. " + "ssh_pwauth=None" + ) in caplog.text + assert [ + mock.call(["systemctl", "status", "ssh"], capture=True) + ] == m_subp.call_args_list - @mock.patch(MODPATH + "subp.subp") - def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp): + @mock.patch(f"{MODPATH}subp.subp") + def test_handle_on_chpasswd_list_parses_common_hashes( + self, _m_subp, caplog + ): """handle parses command password hashes.""" - cloud = self.tmp_cloud(distro="ubuntu") + cloud = get_cloud() valid_hashed_pwds = [ "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqYpUW.BrPx/" "Dlew1Va", @@ -283,119 +292,530 @@ def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp): "SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1", ] cfg = {"chpasswd": {"list": valid_hashed_pwds}} - with mock.patch.object(setpass, "chpasswd") as chpasswd: - setpass.handle( - "IGNORED", cfg=cfg, cloud=cloud, log=self.logger, args=[] - ) - self.assertIn( - "DEBUG: Handling input for chpasswd as list.", self.logs.getvalue() - ) - self.assertIn( - "DEBUG: Setting hashed password for ['root', 'ubuntu']", - self.logs.getvalue(), - ) - valid = "\n".join(valid_hashed_pwds) + "\n" - called = chpasswd.call_args[0][1] - self.assertEqual(valid, called) - - @mock.patch(MODPATH + "util.is_BSD", return_value=True) - @mock.patch(MODPATH + "subp.subp") - def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( - self, m_subp, m_is_bsd - ): - """BSD don't use chpasswd""" - cloud = get_cloud(distro="freebsd") - valid_pwds = ["ubuntu:passw0rd"] - cfg = {"chpasswd": {"list": valid_pwds}} - with mock.patch.object( - cloud.distro, "uses_systemd", return_value=False - ): - setpass.handle( - "IGNORED", cfg=cfg, cloud=cloud, log=self.logger, args=[] - ) - self.assertEqual( - [ - mock.call( - ["pw", "usermod", "ubuntu", "-h", "0"], - data="passw0rd", - logstring="chpasswd for ubuntu", - ), - mock.call(["pw", "usermod", "ubuntu", "-p", "01-Jan-1970"]), - mock.call(["service", "sshd", "status"], capture=True), - ], - m_subp.call_args_list, - ) + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + assert "Handling input for chpasswd as list." in caplog.text + assert "Setting hashed password for ['root', 'ubuntu']" in caplog.text + + first_arg = chpasswd.call_args[0] + for i, val in enumerate(*first_arg): + assert valid_hashed_pwds[i] == ":".join(val) - @mock.patch(MODPATH + "util.multi_log") - @mock.patch(MODPATH + "subp.subp") - def test_handle_on_chpasswd_list_creates_random_passwords( - self, m_subp, m_multi_log + @mock.patch(f"{MODPATH}subp.subp") + def test_handle_on_chpasswd_users_parses_common_hashes( + self, _m_subp, caplog ): + """handle parses command password hashes.""" + cloud = get_cloud() + valid_hashed_pwds = [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqYpUW.BrPx/Dlew1Va", # noqa: E501 + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1", # noqa: E501 + }, + ] + cfg = {"chpasswd": {"users": valid_hashed_pwds}} + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + assert "Handling input for chpasswd as list." not in caplog.text + assert "Setting hashed password for ['root', 'ubuntu']" in caplog.text + first_arg = chpasswd.call_args[0] + for i, (name, password) in enumerate(*first_arg): + assert valid_hashed_pwds[i]["name"] == name + assert valid_hashed_pwds[i]["password"] == password + + @pytest.mark.parametrize( + "user_cfg", + [ + {"expire": "false", "list": ["root:R", "ubuntu:RANDOM"]}, + { + "expire": "false", + "users": [ + { + "name": "root", + "type": "RANDOM", + }, + { + "name": "ubuntu", + "type": "RANDOM", + }, + ], + }, + ], + ) + def test_random_passwords(self, user_cfg, mocker, caplog): """handle parses command set random passwords.""" - cloud = self.tmp_cloud(distro="ubuntu") - valid_random_pwds = ["root:R", "ubuntu:RANDOM"] - cfg = {"chpasswd": {"expire": "false", "list": valid_random_pwds}} - with mock.patch.object(setpass, "chpasswd") as chpasswd: - setpass.handle( - "IGNORED", cfg=cfg, cloud=cloud, log=self.logger, args=[] - ) - self.assertIn( - "DEBUG: Handling input for chpasswd as list.", self.logs.getvalue() - ) - self.assertEqual(1, chpasswd.call_count) - passwords, _ = chpasswd.call_args - user_pass = { - user: password - for user, password in ( - line.split(":") for line in passwords[1].splitlines() - ) - } + m_multi_log = mocker.patch(f"{MODPATH}util.multi_log") + mocker.patch(f"{MODPATH}subp.subp") + + cloud = get_cloud() + cfg = {"chpasswd": user_cfg} + + with mock.patch.object(setpass.Distro, "chpasswd") as chpasswd: + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + dbg_text = "Handling input for chpasswd as list." + if "list" in cfg["chpasswd"]: + assert dbg_text in caplog.text + else: + assert dbg_text not in caplog.text + assert 1 == chpasswd.call_count + user_pass = dict(*chpasswd.call_args[0]) - self.assertEqual(1, m_multi_log.call_count) - self.assertEqual( - mock.call(mock.ANY, stderr=False, fallback_to_stdout=False), - m_multi_log.call_args, + assert 1 == m_multi_log.call_count + assert ( + mock.call(mock.ANY, stderr=False, fallback_to_stdout=False) + == m_multi_log.call_args ) - self.assertEqual(set(["root", "ubuntu"]), set(user_pass.keys())) + assert {"root", "ubuntu"} == set(user_pass.keys()) written_lines = m_multi_log.call_args[0][0].splitlines() for password in user_pass.values(): for line in written_lines: if password in line: break else: - self.fail("Password not emitted to console") + pytest.fail("Password not emitted to console") + + @pytest.mark.parametrize( + "list_def, users_def", + [ + # demonstrate that new addition matches current behavior + ( + { + "chpasswd": { + "list": [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqY" + "pUW.BrPx/Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + "dog:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC" + "7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx" + "3oo1", + "Till:RANDOM", + ] + } + }, + { + "chpasswd": { + "users": [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y" + "5WojbXWqnqYpUW.BrPx/Dlew1Va", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + { + "name": "dog", + "type": "hash", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + { + "name": "Till", + "type": "RANDOM", + }, + ] + } + }, + ), + # Duplicate user: demonstrate no change in current duplicate + # behavior + ( + { + "chpasswd": { + "list": [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqY" + "pUW.BrPx/Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + ] + } + }, + { + "chpasswd": { + "users": [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y" + "5WojbXWqnqYpUW.BrPx/Dlew1Va", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSw" + "OlbOQSW/HpXazGGx3oo1", + }, + ] + } + }, + ), + # Duplicate user: demonstrate duplicate across users/list doesn't + # change + ( + { + "chpasswd": { + "list": [ + "root:$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y5WojbXWqnqY" + "pUW.BrPx/Dlew1Va", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + ] + } + }, + { + "chpasswd": { + "users": [ + { + "name": "root", + "password": "$2y$10$8BQjxjVByHA/Ee.O1bCXtO8S7Y" + "5WojbXWqnqYpUW.BrPx/Dlew1Va", + }, + { + "name": "ubuntu", + "password": "$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9" + "acWCVEoakMMC7dR5" + "2qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx" + "3oo1", + }, + ], + "list": [ + "ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoak" + "MMC7dR52qSDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXaz" + "GGx3oo1", + ], + } + }, + ), + ], + ) + def test_chpasswd_parity(self, list_def, users_def): + """Assert that two different configs cause identical calls""" + + cloud = get_cloud() + + def_1 = get_chpasswd_calls(list_def, cloud, LOG) + def_2 = get_chpasswd_calls(users_def, cloud, LOG) + assert def_1 == def_2 + assert def_1[-1] == mock.call( + ["systemctl", "status", "ssh"], capture=True + ) + for val in def_1: + assert val + + +expire_cases = [ + { + "chpasswd": { + "expire": True, + "list": [ + "user1:password", + "user2:R", + "user3:$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", + ], + } + }, + { + "chpasswd": { + "expire": True, + "users": [ + { + "name": "user1", + "password": "password", + "type": "text", + }, + { + "name": "user2", + "type": "RANDOM", + }, + { + "name": "user3", + "password": "$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", # noqa: E501 + }, + ], + } + }, + { + "chpasswd": { + "expire": False, + "list": [ + "user1:password", + "user2:R", + "user3:$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", + ], + } + }, + { + "chpasswd": { + "expire": False, + "users": [ + { + "name": "user1", + "password": "password", + "type": "text", + }, + { + "name": "user2", + "type": "RANDOM", + }, + { + "name": "user3", + "password": "$6$cTpht$Z2pSYxleRWK8IrsynFzHcrnPlpUhA7N9AM/", # noqa: E501 + }, + ], + } + }, +] + + +class TestExpire: + @pytest.mark.parametrize("cfg", expire_cases) + def test_expire(self, cfg, mocker, caplog): + features.EXPIRE_APPLIES_TO_HASHED_USERS = True + cloud = get_cloud() + mocker.patch(f"{MODPATH}subp.subp") + mocker.patch.object(cloud.distro, "chpasswd") + m_expire = mocker.patch.object(cloud.distro, "expire_passwd") + + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + + if bool(cfg["chpasswd"]["expire"]): + assert m_expire.call_args_list == [ + mock.call("user1"), + mock.call("user2"), + mock.call("user3"), + ] + assert ( + "Expired passwords for: ['user1', 'user2', 'user3'] users" + in caplog.text + ) + else: + assert m_expire.call_args_list == [] + assert "Expired passwords" not in caplog.text + + @pytest.mark.parametrize("cfg", expire_cases) + def test_expire_old_behavior(self, cfg, mocker, caplog): + # Previously expire didn't apply to hashed passwords. + # Ensure we can preserve that case on older releases + features.EXPIRE_APPLIES_TO_HASHED_USERS = False + cloud = get_cloud() + mocker.patch(f"{MODPATH}subp.subp") + mocker.patch.object(cloud.distro, "chpasswd") + m_expire = mocker.patch.object(cloud.distro, "expire_passwd") + + setpass.handle("IGNORED", cfg=cfg, cloud=cloud, log=LOG, args=[]) + + if bool(cfg["chpasswd"]["expire"]): + assert m_expire.call_args_list == [ + mock.call("user1"), + mock.call("user2"), + ] + assert ( + "Expired passwords for: ['user1', 'user2'] users" + in caplog.text + ) + else: + assert m_expire.call_args_list == [] + assert "Expired passwords" not in caplog.text class TestSetPasswordsSchema: @pytest.mark.parametrize( - "config, error_msg", + "config, expectation", [ # Test both formats still work - ({"ssh_pwauth": True}, None), - ({"ssh_pwauth": "yes"}, None), - ({"ssh_pwauth": "unchanged"}, None), - ({"chpasswd": {"list": "blah"}}, None), + ({"ssh_pwauth": True}, does_not_raise()), + ({"ssh_pwauth": False}, does_not_raise()), + ( + {"ssh_pwauth": "yes"}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: ssh_pwauth: DEPRECATED. Use of" + " non-boolean values for this field is DEPRECATED and" + " will result in an error in a future version of" + " cloud-init." + ), + ), + ), + ( + {"ssh_pwauth": "unchanged"}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: ssh_pwauth: DEPRECATED. Use of" + " non-boolean values for this field is DEPRECATED and" + " will result in an error in a future version of" + " cloud-init." + ), + ), + ), + ( + {"chpasswd": {"list": "blah"}}, + pytest.raises(SchemaValidationError, match="DEPRECATED"), + ), + # Valid combinations + ( + { + "chpasswd": { + "users": [ + { + "name": "what-if-1", + "type": "text", + "password": "correct-horse-battery-staple", + }, + { + "name": "what-if-2", + "type": "hash", + "password": "no-magic-parsing-done-here", + }, + { + "name": "what-if-3", + "password": "type-is-optional-default-" + "value-is-hash", + }, + { + "name": "what-if-4", + "type": "RANDOM", + }, + ] + } + }, + does_not_raise(), + ), + ( + { + "chpasswd": { + "users": [ + { + "name": "what-if-1", + "type": "plaintext", + "password": "type-has-two-legal-values: " + "{'hash', 'text'}", + } + ] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + ( + { + "chpasswd": { + "users": [ + { + "name": "what-if-1", + "type": "RANDOM", + "password": "but you want random?", + } + ] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + ( + {"chpasswd": {"users": [{"password": "."}]}}, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + # when type != RANDOM, password is a required key + ( + { + "chpasswd": { + "users": [{"name": "what-if-1", "type": "hash"}] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), + pytest.param( + { + "chpasswd": { + "users": [ + { + "name": "sonata", + "password": "dit", + "dat": "dot", + } + ] + } + }, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + id="dat_is_an_additional_property", + ), + ( + {"chpasswd": {"users": [{"name": "."}]}}, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + ), # Test regex - ({"chpasswd": {"list": ["user:pass"]}}, None), + ( + {"chpasswd": {"list": ["user:pass"]}}, + pytest.raises(SchemaValidationError, match="DEPRECATED"), + ), # Test valid - ({"password": "pass"}, None), + ({"password": "pass"}, does_not_raise()), # Test invalid values ( {"chpasswd": {"expire": "yes"}}, - "'yes' is not of type 'boolean'", + pytest.raises( + SchemaValidationError, + match="'yes' is not of type 'boolean'", + ), + ), + ( + {"chpasswd": {"list": ["user"]}}, + pytest.raises(SchemaValidationError), + ), + ( + {"chpasswd": {"list": []}}, + pytest.raises( + SchemaValidationError, match=r"\[\] is too short" + ), ), - ({"chpasswd": {"list": ["user"]}}, ""), - ({"chpasswd": {"list": []}}, r"\[\] is too short"), ], ) @skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): - if error_msg is None: + def test_schema_validation(self, config, expectation): + with expectation: validate_cloudconfig_schema(config, get_schema(), strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, get_schema(), strict=True) # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_snap.py b/tests/unittests/config/test_cc_snap.py index 855c23fce..432f72cee 100644 --- a/tests/unittests/config/test_cc_snap.py +++ b/tests/unittests/config/test_cc_snap.py @@ -1,28 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging +import os import re from io import StringIO import pytest -from cloudinit import util -from cloudinit.config.cc_snap import ( - ASSERTIONS_FILE, - add_assertions, - handle, - run_commands, -) +from cloudinit import helpers, util +from cloudinit.config.cc_snap import add_assertions, handle, run_commands from cloudinit.config.schema import ( SchemaValidationError, get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - CiTestCase, - mock, - skipUnlessJsonSchema, - wrap_and_call, -) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.util import get_cloud + +M_PATH = "cloudinit.config.cc_snap." +ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" SYSTEM_USER_ASSERTION = """\ type: system-user @@ -91,98 +87,81 @@ oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""" -class FakeCloud(object): - def __init__(self, distro): - self.distro = distro - - -class TestAddAssertions(CiTestCase): - - with_logs = True +@pytest.fixture() +def fake_cloud(tmpdir): + paths = helpers.Paths( + { + "cloud_dir": tmpdir.join("cloud"), + "run_dir": tmpdir.join("cloud-init"), + "templates_dir": tmpdir.join("templates"), + } + ) + cloud = get_cloud(paths=paths) + yield cloud - def setUp(self): - super(TestAddAssertions, self).setUp() - self.tmp = self.tmp_dir() +class TestAddAssertions: @mock.patch("cloudinit.config.cc_snap.subp.subp") - def test_add_assertions_on_empty_list(self, m_subp): + def test_add_assertions_on_empty_list(self, m_subp, caplog, tmpdir): """When provided with an empty list, add_assertions does nothing.""" - add_assertions([]) - self.assertEqual("", self.logs.getvalue()) - m_subp.assert_not_called() + assert_file = tmpdir.join("snapd.assertions") + add_assertions([], assert_file) + assert not caplog.text + assert 0 == m_subp.call_count - def test_add_assertions_on_non_list_or_dict(self): + def test_add_assertions_on_non_list_or_dict(self, tmpdir): """When provided an invalid type, add_assertions raises an error.""" - with self.assertRaises(TypeError) as context_manager: - add_assertions(assertions="I'm Not Valid") - self.assertEqual( - "assertion parameter was not a list or dict: I'm Not Valid", - str(context_manager.exception), - ) + assert_file = tmpdir.join("snapd.assertions") + with pytest.raises( + TypeError, + match="assertion parameter was not a list or dict: I'm Not Valid", + ): + add_assertions("I'm Not Valid", assert_file) @mock.patch("cloudinit.config.cc_snap.subp.subp") - def test_add_assertions_adds_assertions_as_list(self, m_subp): + def test_add_assertions_adds_assertions_as_list( + self, m_subp, caplog, tmpdir + ): """When provided with a list, add_assertions adds all assertions.""" - self.assertEqual( - ASSERTIONS_FILE, "/var/lib/cloud/instance/snapd.assertions" - ) - assert_file = self.tmp_path("snapd.assertions", dir=self.tmp) + assert_file = tmpdir.join("snapd.assertions") assertions = [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION] - wrap_and_call( - "cloudinit.config.cc_snap", - {"ASSERTIONS_FILE": {"new": assert_file}}, - add_assertions, - assertions, - ) - self.assertIn( - "Importing user-provided snap assertions", self.logs.getvalue() - ) - self.assertIn("sertions", self.logs.getvalue()) - self.assertEqual( - [mock.call(["snap", "ack", assert_file], capture=True)], - m_subp.call_args_list, - ) - compare_file = self.tmp_path("comparison", dir=self.tmp) + add_assertions(assertions, assert_file) + assert "Importing user-provided snap assertions" in caplog.text + assert "sertions" in caplog.text + assert [ + mock.call(["snap", "ack", assert_file], capture=True) + ] == m_subp.call_args_list + compare_file = tmpdir.join("comparison") util.write_file(compare_file, "\n".join(assertions).encode("utf-8")) - self.assertEqual( - util.load_file(compare_file), util.load_file(assert_file) - ) + assert util.load_file(compare_file) == util.load_file(assert_file) @mock.patch("cloudinit.config.cc_snap.subp.subp") - def test_add_assertions_adds_assertions_as_dict(self, m_subp): + def test_add_assertions_adds_assertions_as_dict( + self, m_subp, caplog, tmpdir + ): """When provided with a dict, add_assertions adds all assertions.""" - self.assertEqual( - ASSERTIONS_FILE, "/var/lib/cloud/instance/snapd.assertions" - ) - assert_file = self.tmp_path("snapd.assertions", dir=self.tmp) + assert_file = tmpdir.join("snapd.assertions") assertions = {"00": SYSTEM_USER_ASSERTION, "01": ACCOUNT_ASSERTION} - wrap_and_call( - "cloudinit.config.cc_snap", - {"ASSERTIONS_FILE": {"new": assert_file}}, - add_assertions, - assertions, - ) - self.assertIn( - "Importing user-provided snap assertions", self.logs.getvalue() - ) - self.assertIn( - "DEBUG: Snap acking: ['type: system-user', 'authority-id: Lqv", - self.logs.getvalue(), - ) - self.assertIn( - "DEBUG: Snap acking: ['type: account-key', 'authority-id: canonic", - self.logs.getvalue(), - ) - self.assertEqual( - [mock.call(["snap", "ack", assert_file], capture=True)], - m_subp.call_args_list, - ) - compare_file = self.tmp_path("comparison", dir=self.tmp) + add_assertions(assertions, assert_file) + assert "Importing user-provided snap assertions" in caplog.text + assert ( + M_PATH[:-1], + logging.DEBUG, + "Snap acking: ['type: system-user', 'authority-id: " + "LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp']", + ) in caplog.record_tuples + assert ( + M_PATH[:-1], + logging.DEBUG, + "Snap acking: ['type: account-key', 'authority-id: canonical']", + ) in caplog.record_tuples + assert [ + mock.call(["snap", "ack", assert_file], capture=True) + ] == m_subp.call_args_list + compare_file = tmpdir.join("comparison") combined = "\n".join(assertions.values()) util.write_file(compare_file, combined.encode("utf-8")) - self.assertEqual( - util.load_file(compare_file), util.load_file(assert_file) - ) + assert util.load_file(compare_file) == util.load_file(assert_file) class TestRunCommands(CiTestCase): @@ -339,37 +318,21 @@ def test_schema_validation(self, config, error_msg): validate_cloudconfig_schema(config, get_schema(), strict=True) -class TestHandle(CiTestCase): - - with_logs = True - - def setUp(self): - super(TestHandle, self).setUp() - self.tmp = self.tmp_dir() - +class TestHandle: @mock.patch("cloudinit.config.cc_snap.subp.subp") - def test_handle_adds_assertions(self, m_subp): + def test_handle_adds_assertions(self, m_subp, fake_cloud, tmpdir): """Any configured snap assertions are provided to add_assertions.""" - assert_file = self.tmp_path("snapd.assertions", dir=self.tmp) - compare_file = self.tmp_path("comparison", dir=self.tmp) + assert_file = os.path.join( + fake_cloud.paths.get_ipath_cur(), "snapd.assertions" + ) + compare_file = tmpdir.join("comparison") cfg = { "snap": {"assertions": [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]} } - wrap_and_call( - "cloudinit.config.cc_snap", - {"ASSERTIONS_FILE": {"new": assert_file}}, - handle, - "snap", - cfg=cfg, - cloud=None, - log=self.logger, - args=None, - ) + handle("snap", cfg=cfg, cloud=fake_cloud, log=mock.Mock(), args=None) content = "\n".join(cfg["snap"]["assertions"]) util.write_file(compare_file, content.encode("utf-8")) - self.assertEqual( - util.load_file(compare_file), util.load_file(assert_file) - ) + assert util.load_file(compare_file) == util.load_file(assert_file) # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_ssh.py b/tests/unittests/config/test_cc_ssh.py index 47c0c7770..8f2ca8bfe 100644 --- a/tests/unittests/config/test_cc_ssh.py +++ b/tests/unittests/config/test_cc_ssh.py @@ -57,6 +57,7 @@ def _replace_options(user: Optional[str] = None) -> str: return options +@pytest.mark.usefixtures("fake_filesystem") @mock.patch(MODPATH + "ssh_util.setup_user_keys") class TestHandleSsh: """Test cc_ssh handling of ssh config.""" @@ -283,12 +284,30 @@ def test_handle_publish_hostkeys( expected_calls == cloud.datasource.publish_host_keys.call_args_list ) + @pytest.mark.parametrize("with_sshd_dconf", [False, True]) + @mock.patch(MODPATH + "util.ensure_dir") @mock.patch(MODPATH + "ug_util.normalize_users_groups") @mock.patch(MODPATH + "util.write_file") - def test_handle_ssh_keys_in_cfg(self, m_write_file, m_nug, m_setup_keys): + def test_handle_ssh_keys_in_cfg( + self, + m_write_file, + m_nug, + m_ensure_dir, + m_setup_keys, + with_sshd_dconf, + mocker, + ): """Test handle with ssh keys and certificate.""" # Populate a config dictionary to pass to handle() as well # as the expected file-writing calls. + mocker.patch( + MODPATH + "ssh_util._includes_dconf", return_value=with_sshd_dconf + ) + if with_sshd_dconf: + sshd_conf_fname = "/etc/ssh/sshd_config.d/50-cloud-init.conf" + else: + sshd_conf_fname = "/etc/ssh/sshd_config" + cfg = {"ssh_keys": {}} expected_calls = [] @@ -324,7 +343,7 @@ def test_handle_ssh_keys_in_cfg(self, m_write_file, m_nug, m_setup_keys): 384, ), mock.call( - "/etc/ssh/sshd_config", + sshd_conf_fname, "HostCertificate /etc/ssh/ssh_host_{}_key-cert.pub" "\n".format(key_type), preserve_mode=True, @@ -343,6 +362,14 @@ def test_handle_ssh_keys_in_cfg(self, m_write_file, m_nug, m_setup_keys): for call_ in expected_calls: assert call_ in m_write_file.call_args_list + if with_sshd_dconf: + assert ( + mock.call("/etc/ssh/sshd_config.d", mode=0o755) + in m_ensure_dir.call_args_list + ) + else: + assert [] == m_ensure_dir.call_args_list + @pytest.mark.parametrize( "key_type,reason", [ diff --git a/tests/unittests/config/test_cc_ubuntu_advantage.py b/tests/unittests/config/test_cc_ubuntu_advantage.py index 0c5544e13..657bfe519 100644 --- a/tests/unittests/config/test_cc_ubuntu_advantage.py +++ b/tests/unittests/config/test_cc_ubuntu_advantage.py @@ -1,4 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging import re import pytest @@ -8,59 +9,137 @@ configure_ua, handle, maybe_install_ua_tools, + supplemental_schema_validation, ) from cloudinit.config.schema import ( SchemaValidationError, get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import does_not_raise, mock, skipUnlessJsonSchema +from tests.unittests.util import get_cloud # Module path used in mocks MPATH = "cloudinit.config.cc_ubuntu_advantage" -class FakeCloud(object): - def __init__(self, distro): - self.distro = distro - - -class TestConfigureUA(CiTestCase): - - with_logs = True - allowed_subp = [CiTestCase.SUBP_SHELL_TRUE] - - def setUp(self): - super(TestConfigureUA, self).setUp() - self.tmp = self.tmp_dir() - - @mock.patch("%s.subp.subp" % MPATH) +@mock.patch(f"{MPATH}.subp.subp") +class TestConfigureUA: def test_configure_ua_attach_error(self, m_subp): """Errors from ua attach command are raised.""" m_subp.side_effect = subp.ProcessExecutionError( "Invalid token SomeToken" ) - with self.assertRaises(RuntimeError) as context_manager: - configure_ua(token="SomeToken") - self.assertEqual( + match = ( "Failure attaching Ubuntu Advantage:\nUnexpected error while" " running command.\nCommand: -\nExit code: -\nReason: -\n" - "Stdout: Invalid token SomeToken\nStderr: -", - str(context_manager.exception), + "Stdout: Invalid token SomeToken\nStderr: -" ) + with pytest.raises(RuntimeError, match=match): + configure_ua(token="SomeToken") - @mock.patch("%s.subp.subp" % MPATH) - def test_configure_ua_attach_with_token(self, m_subp): - """When token is provided, attach the machine to ua using the token.""" - configure_ua(token="SomeToken") - m_subp.assert_called_once_with(["ua", "attach", "SomeToken"]) - self.assertEqual( - "DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n", - self.logs.getvalue(), - ) + @pytest.mark.parametrize( + "kwargs, call_args_list, log_record_tuples", + [ + # When token is provided, attach the machine to ua using the token. + pytest.param( + {"token": "SomeToken"}, + [mock.call(["ua", "attach", "SomeToken"])], + [ + ( + MPATH, + logging.DEBUG, + "Attaching to Ubuntu Advantage. ua attach SomeToken", + ) + ], + id="with_token", + ), + # When services is an empty list, do not auto-enable attach. + pytest.param( + {"token": "SomeToken", "enable": []}, + [mock.call(["ua", "attach", "SomeToken"])], + [ + ( + MPATH, + logging.DEBUG, + "Attaching to Ubuntu Advantage. ua attach SomeToken", + ) + ], + id="with_empty_services", + ), + # When services a list, only enable specific services. + pytest.param( + {"token": "SomeToken", "enable": ["fips"]}, + [ + mock.call(["ua", "attach", "SomeToken"]), + mock.call( + ["ua", "enable", "--assume-yes", "fips"], capture=True + ), + ], + [ + ( + MPATH, + logging.DEBUG, + "Attaching to Ubuntu Advantage. ua attach SomeToken", + ) + ], + id="with_specific_services", + ), + # When services a string, treat as singleton list and warn + pytest.param( + {"token": "SomeToken", "enable": "fips"}, + [ + mock.call(["ua", "attach", "SomeToken"]), + mock.call( + ["ua", "enable", "--assume-yes", "fips"], capture=True + ), + ], + [ + ( + MPATH, + logging.DEBUG, + "Attaching to Ubuntu Advantage. ua attach SomeToken", + ), + ( + MPATH, + logging.WARNING, + "ubuntu_advantage: enable should be a list, not a " + "string; treating as a single enable", + ), + ], + id="with_string_services", + ), + # When services not string or list, warn but still attach + pytest.param( + {"token": "SomeToken", "enable": {"deffo": "wont work"}}, + [mock.call(["ua", "attach", "SomeToken"])], + [ + ( + MPATH, + logging.DEBUG, + "Attaching to Ubuntu Advantage. ua attach SomeToken", + ), + ( + MPATH, + logging.WARNING, + "ubuntu_advantage: enable should be a list, not a" + " dict; skipping enabling services", + ), + ], + id="with_weird_services", + ), + ], + ) + @mock.patch(f"{MPATH}.maybe_install_ua_tools", mock.MagicMock()) + def test_configure_ua_attach( + self, m_subp, kwargs, call_args_list, log_record_tuples, caplog + ): + configure_ua(**kwargs) + assert call_args_list == m_subp.call_args_list + for record_tuple in log_record_tuples: + assert record_tuple in caplog.record_tuples - @mock.patch("%s.subp.subp" % MPATH) - def test_configure_ua_attach_on_service_error(self, m_subp): + def test_configure_ua_attach_on_service_error(self, m_subp, caplog): """all services should be enabled and then any failures raised""" def fake_subp(cmd, capture=None): @@ -75,102 +154,86 @@ def fake_subp(cmd, capture=None): m_subp.side_effect = fake_subp - with self.assertRaises(RuntimeError) as context_manager: + with pytest.raises( + RuntimeError, + match=re.escape( + 'Failure enabling Ubuntu Advantage service(s): "esm", "cc"' + ), + ): configure_ua(token="SomeToken", enable=["esm", "cc", "fips"]) - self.assertEqual( - m_subp.call_args_list, - [ - mock.call(["ua", "attach", "SomeToken"]), - mock.call( - ["ua", "enable", "--assume-yes", "esm"], capture=True - ), - mock.call( - ["ua", "enable", "--assume-yes", "cc"], capture=True - ), - mock.call( - ["ua", "enable", "--assume-yes", "fips"], capture=True - ), - ], - ) - self.assertIn( - 'WARNING: Failure enabling "esm":\nUnexpected error' + assert m_subp.call_args_list == [ + mock.call(["ua", "attach", "SomeToken"]), + mock.call(["ua", "enable", "--assume-yes", "esm"], capture=True), + mock.call(["ua", "enable", "--assume-yes", "cc"], capture=True), + mock.call(["ua", "enable", "--assume-yes", "fips"], capture=True), + ] + assert ( + MPATH, + logging.WARNING, + 'Failure enabling "esm":\nUnexpected error' " while running command.\nCommand: -\nExit code: -\nReason: -\n" - "Stdout: Invalid ESM credentials\nStderr: -\n", - self.logs.getvalue(), - ) - self.assertIn( - 'WARNING: Failure enabling "cc":\nUnexpected error' + "Stdout: Invalid ESM credentials\nStderr: -", + ) in caplog.record_tuples + assert ( + MPATH, + logging.WARNING, + 'Failure enabling "cc":\nUnexpected error' " while running command.\nCommand: -\nExit code: -\nReason: -\n" - "Stdout: Invalid CC credentials\nStderr: -\n", - self.logs.getvalue(), - ) - self.assertEqual( - 'Failure enabling Ubuntu Advantage service(s): "esm", "cc"', - str(context_manager.exception), + "Stdout: Invalid CC credentials\nStderr: -", + ) in caplog.record_tuples + assert 'Failure enabling "fips"' not in caplog.text + + def test_configure_ua_config_with_weird_params(self, m_subp, caplog): + """When configs not string or list, warn but still attach""" + configure_ua( + token="SomeToken", config=["http_proxy=http://some-proxy.net:3128"] ) - - @mock.patch("%s.subp.subp" % MPATH) - def test_configure_ua_attach_with_empty_services(self, m_subp): - """When services is an empty list, do not auto-enable attach.""" - configure_ua(token="SomeToken", enable=[]) - m_subp.assert_called_once_with(["ua", "attach", "SomeToken"]) - self.assertEqual( - "DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n", - self.logs.getvalue(), - ) - - @mock.patch("%s.subp.subp" % MPATH) - def test_configure_ua_attach_with_specific_services(self, m_subp): - """When services a list, only enable specific services.""" - configure_ua(token="SomeToken", enable=["fips"]) - self.assertEqual( - m_subp.call_args_list, - [ - mock.call(["ua", "attach", "SomeToken"]), - mock.call( - ["ua", "enable", "--assume-yes", "fips"], capture=True - ), - ], - ) - self.assertEqual( - "DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n", - self.logs.getvalue(), - ) - - @mock.patch("%s.maybe_install_ua_tools" % MPATH, mock.MagicMock()) - @mock.patch("%s.subp.subp" % MPATH) - def test_configure_ua_attach_with_string_services(self, m_subp): - """When services a string, treat as singleton list and warn""" - configure_ua(token="SomeToken", enable="fips") - self.assertEqual( - m_subp.call_args_list, - [ - mock.call(["ua", "attach", "SomeToken"]), - mock.call( - ["ua", "enable", "--assume-yes", "fips"], capture=True - ), - ], - ) - self.assertEqual( - "WARNING: ubuntu_advantage: enable should be a list, not a" - " string; treating as a single enable\n" - "DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n", - self.logs.getvalue(), + assert [ + mock.call(["ua", "attach", "SomeToken"]) + ] == m_subp.call_args_list + assert ( + MPATH, + logging.WARNING, + "ubuntu_advantage: config should be a dict, not a" + " list; skipping enabling config parameters", + ) == caplog.record_tuples[-2] + assert ( + MPATH, + logging.DEBUG, + "Attaching to Ubuntu Advantage. ua attach SomeToken", + ) == caplog.record_tuples[-1] + + def test_configure_ua_config_error_invalid_url(self, m_subp, caplog): + """Errors from ua config command are raised.""" + m_subp.side_effect = subp.ProcessExecutionError( + 'Failure enabling "http_proxy"' ) + with pytest.raises( + RuntimeError, + match=re.escape( + 'Failure enabling Ubuntu Advantage config(s): "http_proxy"' + ), + ): + configure_ua( + token="SomeToken", config={"http_proxy": "not-a-valid-url"} + ) - @mock.patch("%s.subp.subp" % MPATH) - def test_configure_ua_attach_with_weird_services(self, m_subp): - """When services not string or list, warn but still attach""" - configure_ua(token="SomeToken", enable={"deffo": "wont work"}) - self.assertEqual( - m_subp.call_args_list, [mock.call(["ua", "attach", "SomeToken"])] - ) - self.assertEqual( - "WARNING: ubuntu_advantage: enable should be a list, not a" - " dict; skipping enabling services\n" - "DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n", - self.logs.getvalue(), + def test_configure_ua_config_error_non_string_values(self, m_subp): + """ValueError raised for any values expected as string type.""" + cfg = { + "global_apt_http_proxy": "noscheme", + "http_proxy": ["no-proxy"], + "https_proxy": 1, + } + match = re.escape( + "Expected URL scheme http/https for" + " ua:config:global_apt_http_proxy. Found: noscheme\n" + "Expected a URL for ua:config:http_proxy. Found: ['no-proxy']\n" + "Expected a URL for ua:config:https_proxy. Found: 1" ) + with pytest.raises(ValueError, match=match): + supplemental_schema_validation(cfg) + assert 0 == m_subp.call_count class TestUbuntuAdvantageSchema: @@ -197,160 +260,223 @@ def test_schema_validation(self, config, error_msg): validate_cloudconfig_schema(config, get_schema(), strict=True) -class TestHandle(CiTestCase): - - with_logs = True - - def setUp(self): - super(TestHandle, self).setUp() - self.tmp = self.tmp_dir() - - @mock.patch("%s.maybe_install_ua_tools" % MPATH) - def test_handle_no_config(self, m_maybe_install_ua_tools): - """When no ua-related configuration is provided, nothing happens.""" - cfg = {} - handle("ua-test", cfg=cfg, cloud=None, log=self.logger, args=None) - self.assertIn( - "DEBUG: Skipping module named ua-test, no 'ubuntu_advantage'" - " configuration found", - self.logs.getvalue(), - ) - self.assertEqual(m_maybe_install_ua_tools.call_count, 0) +class TestHandle: - @mock.patch("%s.configure_ua" % MPATH) - @mock.patch("%s.maybe_install_ua_tools" % MPATH) - def test_handle_tries_to_install_ubuntu_advantage_tools( - self, m_install, m_cfg - ): - """If ubuntu_advantage is provided, try installing ua-tools package.""" - cfg = {"ubuntu_advantage": {"token": "valid"}} - mycloud = FakeCloud(None) - handle("nomatter", cfg=cfg, cloud=mycloud, log=self.logger, args=None) - m_install.assert_called_once_with(mycloud) + cloud = get_cloud() - @mock.patch("%s.configure_ua" % MPATH) - @mock.patch("%s.maybe_install_ua_tools" % MPATH) - def test_handle_passes_credentials_and_services_to_configure_ua( - self, m_install, m_configure_ua + @pytest.mark.parametrize( + [ + "cfg", + "cloud", + "log_record_tuples", + "maybe_install_call_args_list", + "configure_ua_call_args_list", + ], + [ + # When no ua-related configuration is provided, nothing happens. + pytest.param( + {}, + None, + [ + ( + MPATH, + logging.DEBUG, + "Skipping module named nomatter, no 'ubuntu_advantage'" + " configuration found", + ) + ], + [], + [], + id="no_config", + ), + # If ubuntu_advantage is provided, try installing ua-tools package. + pytest.param( + {"ubuntu_advantage": {"token": "valid"}}, + cloud, + [], + [mock.call(cloud)], + None, + id="tries_to_install_ubuntu_advantage_tools", + ), + # All ubuntu_advantage config keys are passed to configure_ua. + pytest.param( + {"ubuntu_advantage": {"token": "token", "enable": ["esm"]}}, + cloud, + [], + [mock.call(cloud)], + [mock.call(token="token", enable=["esm"], config=None)], + id="passes_credentials_and_services_to_configure_ua", + ), + # Warning when ubuntu-advantage key is present with new config + pytest.param( + {"ubuntu-advantage": {"token": "token", "enable": ["esm"]}}, + None, + [ + ( + MPATH, + logging.WARNING, + 'Deprecated configuration key "ubuntu-advantage"' + " provided. Expected underscore delimited " + '"ubuntu_advantage"; will attempt to continue.', + ) + ], + None, + [mock.call(token="token", enable=["esm"], config=None)], + id="warns_on_deprecated_ubuntu_advantage_key_w_config", + ), + # ubuntu_advantage should be preferred over ubuntu-advantage + pytest.param( + { + "ubuntu-advantage": {"token": "nope", "enable": ["wrong"]}, + "ubuntu_advantage": {"token": "token", "enable": ["esm"]}, + }, + None, + [ + ( + MPATH, + logging.WARNING, + 'Deprecated configuration key "ubuntu-advantage"' + " provided. Expected underscore delimited " + '"ubuntu_advantage"; will attempt to continue.', + ) + ], + None, + [mock.call(token="token", enable=["esm"], config=None)], + id="prefers_new_style_config", + ), + ], + ) + @mock.patch(f"{MPATH}.configure_ua") + @mock.patch(f"{MPATH}.maybe_install_ua_tools") + def test_handle( + self, + m_maybe_install_ua_tools, + m_configure_ua, + cfg, + cloud, + log_record_tuples, + maybe_install_call_args_list, + configure_ua_call_args_list, + caplog, ): - """All ubuntu_advantage config keys are passed to configure_ua.""" - cfg = {"ubuntu_advantage": {"token": "token", "enable": ["esm"]}} - handle("nomatter", cfg=cfg, cloud=None, log=self.logger, args=None) - m_configure_ua.assert_called_once_with(token="token", enable=["esm"]) + handle("nomatter", cfg=cfg, cloud=cloud, log=None, args=None) + for record_tuple in log_record_tuples: + assert record_tuple in caplog.record_tuples + if maybe_install_call_args_list is not None: + assert ( + maybe_install_call_args_list + == m_maybe_install_ua_tools.call_args_list + ) + if configure_ua_call_args_list is not None: + assert configure_ua_call_args_list == m_configure_ua.call_args_list - @mock.patch("%s.maybe_install_ua_tools" % MPATH, mock.MagicMock()) + @pytest.mark.parametrize( + "cfg, handle_kwargs, match", + [ + pytest.param( + {"ubuntu-advantage": {"commands": "nogo"}}, + dict(cloud=None, args=None), + ( + 'Deprecated configuration "ubuntu-advantage: commands" ' + 'provided. Expected "token"' + ), + id="key_dashed", + ), + pytest.param( + {"ubuntu_advantage": {"commands": "nogo"}}, + dict(cloud=None, args=None), + ( + 'Deprecated configuration "ubuntu-advantage: commands" ' + 'provided. Expected "token"' + ), + id="key_underscore", + ), + ], + ) @mock.patch("%s.configure_ua" % MPATH) - def test_handle_warns_on_deprecated_ubuntu_advantage_key_w_config( - self, m_configure_ua + def test_handle_error_on_deprecated_commands_key_dashed( + self, m_configure_ua, cfg, handle_kwargs, match ): - """Warning when ubuntu-advantage key is present with new config""" - cfg = {"ubuntu-advantage": {"token": "token", "enable": ["esm"]}} - handle("nomatter", cfg=cfg, cloud=None, log=self.logger, args=None) - self.assertEqual( - 'WARNING: Deprecated configuration key "ubuntu-advantage"' - ' provided. Expected underscore delimited "ubuntu_advantage";' - " will attempt to continue.", - self.logs.getvalue().splitlines()[0], - ) - m_configure_ua.assert_called_once_with(token="token", enable=["esm"]) - - def test_handle_error_on_deprecated_commands_key_dashed(self): - """Error when commands is present in ubuntu-advantage key.""" - cfg = {"ubuntu-advantage": {"commands": "nogo"}} - with self.assertRaises(RuntimeError) as context_manager: - handle("nomatter", cfg=cfg, cloud=None, log=self.logger, args=None) - self.assertEqual( - 'Deprecated configuration "ubuntu-advantage: commands" provided.' - ' Expected "token"', - str(context_manager.exception), - ) + with pytest.raises(RuntimeError, match=match): + handle("nomatter", cfg=cfg, log=mock.Mock(), **handle_kwargs) + assert 0 == m_configure_ua.call_count - def test_handle_error_on_deprecated_commands_key_underscored(self): - """Error when commands is present in ubuntu_advantage key.""" - cfg = {"ubuntu_advantage": {"commands": "nogo"}} - with self.assertRaises(RuntimeError) as context_manager: - handle("nomatter", cfg=cfg, cloud=None, log=self.logger, args=None) - self.assertEqual( - 'Deprecated configuration "ubuntu-advantage: commands" provided.' - ' Expected "token"', - str(context_manager.exception), - ) - - @mock.patch("%s.maybe_install_ua_tools" % MPATH, mock.MagicMock()) - @mock.patch("%s.configure_ua" % MPATH) - def test_handle_prefers_new_style_config(self, m_configure_ua): - """ubuntu_advantage should be preferred over ubuntu-advantage""" - cfg = { - "ubuntu-advantage": {"token": "nope", "enable": ["wrong"]}, - "ubuntu_advantage": {"token": "token", "enable": ["esm"]}, - } - handle("nomatter", cfg=cfg, cloud=None, log=self.logger, args=None) - self.assertEqual( - 'WARNING: Deprecated configuration key "ubuntu-advantage"' - ' provided. Expected underscore delimited "ubuntu_advantage";' - " will attempt to continue.", - self.logs.getvalue().splitlines()[0], - ) - m_configure_ua.assert_called_once_with(token="token", enable=["esm"]) +@mock.patch(f"{MPATH}.subp.which") +class TestMaybeInstallUATools: + @pytest.mark.parametrize( + [ + "which_return", + "update_side_effect", + "install_side_effect", + "expectation", + "log_msg", + ], + [ + # Do nothing if ubuntu-advantage-tools already exists. + pytest.param( + "/usr/bin/ua", # already installed + RuntimeError("Some apt error"), + None, + does_not_raise(), # No RuntimeError + None, + id="noop_when_ua_tools_present", + ), + # logs and raises apt update errors + pytest.param( + None, + RuntimeError("Some apt error"), + None, + pytest.raises(RuntimeError, match="Some apt error"), + "Package update failed\nTraceback", + id="raises_update_errors", + ), + # logs and raises package install errors + pytest.param( + None, + None, + RuntimeError("Some install error"), + pytest.raises(RuntimeError, match="Some install error"), + "Failed to install ubuntu-advantage-tools\n", + id="raises_install_errors", + ), + ], + ) + def test_maybe_install_ua_tools( + self, + m_which, + which_return, + update_side_effect, + install_side_effect, + expectation, + log_msg, + caplog, + ): + m_which.return_value = which_return + cloud = mock.MagicMock() + if install_side_effect is None: + cloud.distro.update_package_sources.side_effect = ( + update_side_effect + ) + else: + cloud.distro.update_package_sources.return_value = None + cloud.distro.install_packages.side_effect = install_side_effect + with expectation: + maybe_install_ua_tools(cloud=cloud) + if log_msg is not None: + assert log_msg in caplog.text -class TestMaybeInstallUATools(CiTestCase): - - with_logs = True - - def setUp(self): - super(TestMaybeInstallUATools, self).setUp() - self.tmp = self.tmp_dir() - - @mock.patch("%s.subp.which" % MPATH) - def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): - """Do nothing if ubuntu-advantage-tools already exists.""" - m_which.return_value = "/usr/bin/ua" # already installed - distro = mock.MagicMock() - distro.update_package_sources.side_effect = RuntimeError( - "Some apt error" - ) - maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError - - @mock.patch("%s.subp.which" % MPATH) - def test_maybe_install_ua_tools_raises_update_errors(self, m_which): - """maybe_install_ua_tools logs and raises apt update errors.""" - m_which.return_value = None - distro = mock.MagicMock() - distro.update_package_sources.side_effect = RuntimeError( - "Some apt error" - ) - with self.assertRaises(RuntimeError) as context_manager: - maybe_install_ua_tools(cloud=FakeCloud(distro)) - self.assertEqual("Some apt error", str(context_manager.exception)) - self.assertIn("Package update failed\nTraceback", self.logs.getvalue()) - - @mock.patch("%s.subp.which" % MPATH) - def test_maybe_install_ua_raises_install_errors(self, m_which): - """maybe_install_ua_tools logs and raises package install errors.""" - m_which.return_value = None - distro = mock.MagicMock() - distro.update_package_sources.return_value = None - distro.install_packages.side_effect = RuntimeError( - "Some install error" - ) - with self.assertRaises(RuntimeError) as context_manager: - maybe_install_ua_tools(cloud=FakeCloud(distro)) - self.assertEqual("Some install error", str(context_manager.exception)) - self.assertIn( - "Failed to install ubuntu-advantage-tools\n", self.logs.getvalue() - ) - - @mock.patch("%s.subp.which" % MPATH) def test_maybe_install_ua_tools_happy_path(self, m_which): """maybe_install_ua_tools installs ubuntu-advantage-tools.""" m_which.return_value = None - distro = mock.MagicMock() # No errors raised - maybe_install_ua_tools(cloud=FakeCloud(distro)) - distro.update_package_sources.assert_called_once_with() - distro.install_packages.assert_called_once_with( - ["ubuntu-advantage-tools"] - ) + cloud = mock.MagicMock() # No errors raised + maybe_install_ua_tools(cloud=cloud) + assert [ + mock.call() + ] == cloud.distro.update_package_sources.call_args_list + assert [ + mock.call(["ubuntu-advantage-tools"]) + ] == cloud.distro.install_packages.call_args_list # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_ubuntu_autoinstall.py b/tests/unittests/config/test_cc_ubuntu_autoinstall.py new file mode 100644 index 000000000..87f44f82f --- /dev/null +++ b/tests/unittests/config/test_cc_ubuntu_autoinstall.py @@ -0,0 +1,141 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import logging +from unittest import mock + +import pytest + +from cloudinit.config import cc_ubuntu_autoinstall +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema +from tests.unittests.util import get_cloud + +LOG = logging.getLogger(__name__) + +MODPATH = "cloudinit.config.cc_ubuntu_autoinstall." + +SAMPLE_SNAP_LIST_OUTPUT = """ +Name Version Rev Tracking ... +core20 20220527 1518 latest/stable ... +lxd git-69dc707 23315 latest/edge ... +""" +SAMPLE_SNAP_LIST_SUBIQUITY = ( + SAMPLE_SNAP_LIST_OUTPUT + + """ +subiquity 22.06.01 23315 latest/stable ... +""" +) +SAMPLE_SNAP_LIST_DESKTOP_INSTALLER = ( + SAMPLE_SNAP_LIST_OUTPUT + + """ +ubuntu-desktop-installer 22.06.01 23315 latest/stable ... +""" +) + + +class TestvalidateConfigSchema: + @pytest.mark.parametrize( + "src_cfg,error_msg", + [ + pytest.param( + {"autoinstall": 1}, + "autoinstall: Expected dict type but found: int", + id="err_non_dict", + ), + pytest.param( + {"autoinstall": {}}, + "autoinstall: Missing required 'version' key", + id="err_require_version_key", + ), + pytest.param( + {"autoinstall": {"version": "v1"}}, + "autoinstall.version: Expected int type but found: str", + id="err_version_non_int", + ), + ], + ) + def test_runtime_validation_errors(self, src_cfg, error_msg): + """cloud-init raises errors at runtime on invalid autoinstall config""" + with pytest.raises(SchemaValidationError, match=error_msg): + cc_ubuntu_autoinstall.validate_config_schema(src_cfg) + + +@mock.patch(MODPATH + "subp") +class TestHandleAutoinstall: + """Test cc_ubuntu_autoinstall handling of config.""" + + @pytest.mark.parametrize( + "cfg,snap_list,subp_calls,logs", + [ + pytest.param( + {}, + SAMPLE_SNAP_LIST_OUTPUT, + [], + ["Skipping module named name, no 'autoinstall' key"], + id="skip_no_cfg", + ), + pytest.param( + {"autoinstall": {"version": 1}}, + SAMPLE_SNAP_LIST_OUTPUT, + [mock.call(["snap", "list"])], + [ + "Skipping autoinstall module. Expected one of the Ubuntu" + " installer snap packages to be present: subiquity," + " ubuntu-desktop-installer" + ], + id="valid_autoinstall_schema_checks_snaps", + ), + pytest.param( + {"autoinstall": {"version": 1}}, + SAMPLE_SNAP_LIST_SUBIQUITY, + [mock.call(["snap", "list"])], + [ + "Valid autoinstall schema. Config will be processed by" + " subiquity" + ], + id="valid_autoinstall_schema_sees_subiquity", + ), + pytest.param( + {"autoinstall": {"version": 1}}, + SAMPLE_SNAP_LIST_DESKTOP_INSTALLER, + [mock.call(["snap", "list"])], + [ + "Valid autoinstall schema. Config will be processed by" + " ubuntu-desktop-installer" + ], + id="valid_autoinstall_schema_sees_desktop_installer", + ), + ], + ) + def test_handle_autoinstall_cfg( + self, subp, cfg, snap_list, subp_calls, logs, caplog + ): + subp.return_value = snap_list, "" + cloud = get_cloud(distro="ubuntu") + cc_ubuntu_autoinstall.handle("name", cfg, cloud, LOG, None) + assert subp_calls == subp.call_args_list + for log in logs: + assert log in caplog.text + + +class TestAutoInstallSchema: + @pytest.mark.parametrize( + "config, error_msg", + ( + ( + {"autoinstall": {}}, + "autoinstall: 'version' is a required property", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_ubuntu_drivers.py b/tests/unittests/config/test_cc_ubuntu_drivers.py index 3cbde8b20..9d54467e9 100644 --- a/tests/unittests/config/test_cc_ubuntu_drivers.py +++ b/tests/unittests/config/test_cc_ubuntu_drivers.py @@ -6,6 +6,7 @@ import pytest +from cloudinit import log from cloudinit.config import cc_ubuntu_drivers as drivers from cloudinit.config.schema import ( SchemaValidationError, @@ -13,7 +14,7 @@ validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" @@ -31,223 +32,286 @@ # pylint: disable=no-value-for-parameter -class AnyTempScriptAndDebconfFile(object): - def __init__(self, tmp_dir, debconf_file): - self.tmp_dir = tmp_dir - self.debconf_file = debconf_file - - def __eq__(self, cmd): - if not len(cmd) == 2: - return False - script, debconf_file = cmd - if bool(script.startswith(self.tmp_dir) and script.endswith(".sh")): - return debconf_file == self.debconf_file - return False - - -class TestUbuntuDrivers(CiTestCase): - cfg_accepted = {"drivers": {"nvidia": {"license-accepted": True}}} +@pytest.mark.parametrize( + "cfg_accepted,install_gpgpu", + [ + pytest.param( + {"drivers": {"nvidia": {"license-accepted": True}}}, + ["ubuntu-drivers", "install", "--gpgpu", "nvidia"], + id="without_version", + ), + pytest.param( + { + "drivers": { + "nvidia": {"license-accepted": True, "version": "123"} + } + }, + ["ubuntu-drivers", "install", "--gpgpu", "nvidia:123"], + id="with_version", + ), + ], +) +@mock.patch(MPATH + "debconf") +@mock.patch(MPATH + "HAS_DEBCONF", True) +class TestUbuntuDrivers: install_gpgpu = ["ubuntu-drivers", "install", "--gpgpu", "nvidia"] - with_logs = True - + @pytest.mark.parametrize( + "true_value", + [ + True, + "yes", + "true", + "on", + "1", + ], + ) @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=False) - def _assert_happy_path_taken(self, config, m_which, m_subp, m_tmp): + def test_happy_path_taken( + self, + m_which, + m_subp, + m_tmp, + m_debconf, + tmpdir, + cfg_accepted, + install_gpgpu, + true_value, + ): """Positive path test through handle. Package should be installed.""" - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") + new_config: dict = copy.deepcopy(cfg_accepted) + new_config["drivers"]["nvidia"]["license-accepted"] = true_value + + tdir = tmpdir + debconf_file = tdir.join("nvidia.template") m_tmp.return_value = tdir myCloud = mock.MagicMock() - drivers.handle("ubuntu_drivers", config, myCloud, None, None) - self.assertEqual( - [mock.call(["ubuntu-drivers-common"])], - myCloud.distro.install_packages.call_args_list, - ) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, - ) - - def test_handle_does_package_install(self): - self._assert_happy_path_taken(self.cfg_accepted) - - def test_trueish_strings_are_considered_approval(self): - for true_value in ["yes", "true", "on", "1"]: - new_config = copy.deepcopy(self.cfg_accepted) - new_config["drivers"]["nvidia"]["license-accepted"] = true_value - self._assert_happy_path_taken(new_config) + drivers.handle("ubuntu_drivers", new_config, myCloud, None, None) + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers-common"]) + ] == myCloud.distro.install_packages.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp") @mock.patch(MPATH + "subp.which", return_value=False) def test_handle_raises_error_if_no_drivers_found( - self, m_which, m_subp, m_tmp + self, + m_which, + m_subp, + m_tmp, + m_debconf, + caplog, + tmpdir, + cfg_accepted, + install_gpgpu, ): """If ubuntu-drivers doesn't install any drivers, raise an error.""" - tdir = self.tmp_dir() + tdir = tmpdir debconf_file = os.path.join(tdir, "nvidia.template") m_tmp.return_value = tdir myCloud = mock.MagicMock() - def fake_subp(cmd): - if cmd[0].startswith(tdir): - return - raise ProcessExecutionError( - stdout="No drivers found for installation.\n", exit_code=1 - ) - - m_subp.side_effect = fake_subp - - with self.assertRaises(Exception): - drivers.handle( - "ubuntu_drivers", self.cfg_accepted, myCloud, None, None - ) - self.assertEqual( - [mock.call(["ubuntu-drivers-common"])], - myCloud.distro.install_packages.call_args_list, - ) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, + m_subp.side_effect = ProcessExecutionError( + stdout="No drivers found for installation.\n", exit_code=1 ) - self.assertIn( - "ubuntu-drivers found no drivers for installation", - self.logs.getvalue(), + + with pytest.raises(Exception): + drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, None, None) + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers-common"]) + ] == myCloud.distro.install_packages.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list + assert ( + "ubuntu-drivers found no drivers for installation" in caplog.text ) + @pytest.mark.parametrize( + "config", + [ + pytest.param( + {"drivers": {"nvidia": {"license-accepted": False}}}, + id="license_not_accepted", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "garbage"}}}, + id="garbage_in_license_field", + ), + pytest.param({"drivers": {"nvidia": {}}}, id="no_license_key"), + pytest.param( + {"drivers": {"acme": {"license-accepted": True}}}, + id="no_nvidia_key", + ), + # ensure we don't do anything if string refusal given + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "no"}}}, + id="string_given_no", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "false"}}}, + id="string_given_false", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "off"}}}, + id="string_given_off", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "0"}}}, + id="string_given_0", + ), + # specifying_a_version_doesnt_override_license_acceptance + pytest.param( + { + "drivers": { + "nvidia": {"license-accepted": False, "version": "123"} + } + }, + id="with_version", + ), + ], + ) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=False) - def _assert_inert_with_config(self, config, m_which, m_subp): + def test_handle_inert( + self, m_which, m_subp, m_debconf, cfg_accepted, install_gpgpu, config + ): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() drivers.handle("ubuntu_drivers", config, myCloud, None, None) - self.assertEqual(0, myCloud.distro.install_packages.call_count) - self.assertEqual(0, m_subp.call_count) - - def test_handle_inert_if_license_not_accepted(self): - """Ensure we don't do anything if the license is rejected.""" - self._assert_inert_with_config( - {"drivers": {"nvidia": {"license-accepted": False}}} - ) - - def test_handle_inert_if_garbage_in_license_field(self): - """Ensure we don't do anything if unknown text is in license field.""" - self._assert_inert_with_config( - {"drivers": {"nvidia": {"license-accepted": "garbage"}}} - ) - - def test_handle_inert_if_no_license_key(self): - """Ensure we don't do anything if no license key.""" - self._assert_inert_with_config({"drivers": {"nvidia": {}}}) - - def test_handle_inert_if_no_nvidia_key(self): - """Ensure we don't do anything if other license accepted.""" - self._assert_inert_with_config( - {"drivers": {"acme": {"license-accepted": True}}} - ) - - def test_handle_inert_if_string_given(self): - """Ensure we don't do anything if string refusal given.""" - for false_value in ["no", "false", "off", "0"]: - self._assert_inert_with_config( - {"drivers": {"nvidia": {"license-accepted": false_value}}} - ) + assert 0 == myCloud.distro.install_packages.call_count + assert 0 == m_subp.call_count @mock.patch(MPATH + "install_drivers") - def test_handle_no_drivers_does_nothing(self, m_install_drivers): + def test_handle_no_drivers_does_nothing( + self, m_install_drivers, m_debconf, cfg_accepted, install_gpgpu + ): """If no 'drivers' key in the config, nothing should be done.""" myCloud = mock.MagicMock() myLog = mock.MagicMock() drivers.handle("ubuntu_drivers", {"foo": "bzr"}, myCloud, myLog, None) - self.assertIn( - "Skipping module named", myLog.debug.call_args_list[0][0][0] - ) - self.assertEqual(0, m_install_drivers.call_count) + assert "Skipping module named" in myLog.debug.call_args_list[0][0][0] + assert 0 == m_install_drivers.call_count @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=True) def test_install_drivers_no_install_if_present( - self, m_which, m_subp, m_tmp + self, + m_which, + m_subp, + m_tmp, + m_debconf, + tmpdir, + cfg_accepted, + install_gpgpu, ): """If 'ubuntu-drivers' is present, no package install should occur.""" - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") + tdir = tmpdir + debconf_file = tmpdir.join("nvidia.template") m_tmp.return_value = tdir pkg_install = mock.MagicMock() drivers.install_drivers( - self.cfg_accepted["drivers"], pkg_install_func=pkg_install + cfg_accepted["drivers"], pkg_install_func=pkg_install ) - self.assertEqual(0, pkg_install.call_count) - self.assertEqual([mock.call("ubuntu-drivers")], m_which.call_args_list) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, - ) - - def test_install_drivers_rejects_invalid_config(self): + assert 0 == pkg_install.call_count + assert [mock.call("ubuntu-drivers")] == m_which.call_args_list + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list + + def test_install_drivers_rejects_invalid_config( + self, m_debconf, cfg_accepted, install_gpgpu + ): """install_drivers should raise TypeError if not given a config dict""" pkg_install = mock.MagicMock() - with self.assertRaisesRegex(TypeError, ".*expected dict.*"): + with pytest.raises(TypeError, match=".*expected dict.*"): drivers.install_drivers("mystring", pkg_install_func=pkg_install) - self.assertEqual(0, pkg_install.call_count) + assert 0 == pkg_install.call_count @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp") @mock.patch(MPATH + "subp.which", return_value=False) def test_install_drivers_handles_old_ubuntu_drivers_gracefully( - self, m_which, m_subp, m_tmp + self, + m_which, + m_subp, + m_tmp, + m_debconf, + caplog, + tmpdir, + cfg_accepted, + install_gpgpu, ): """Older ubuntu-drivers versions should emit message and raise error""" - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") - m_tmp.return_value = tdir + debconf_file = tmpdir.join("nvidia.template") + m_tmp.return_value = tmpdir myCloud = mock.MagicMock() - def fake_subp(cmd): - if cmd[0].startswith(tdir): - return - raise ProcessExecutionError( - stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2 - ) + m_subp.side_effect = ProcessExecutionError( + stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2 + ) - m_subp.side_effect = fake_subp + with pytest.raises(Exception): + drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, None, None) + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers-common"]) + ] == myCloud.distro.install_packages.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list + assert ( + MPATH[:-1], + log.WARNING, + ( + "the available version of ubuntu-drivers is" + " too old to perform requested driver installation" + ), + ) == caplog.record_tuples[-1] - with self.assertRaises(Exception): + @mock.patch(M_TMP_PATH) + @mock.patch(MPATH + "subp.subp", return_value=("", "")) + @mock.patch(MPATH + "subp.which", return_value=False) + def test_debconf_not_installed_does_nothing( + self, + m_which, + m_subp, + m_tmp, + m_debconf, + tmpdir, + cfg_accepted, + install_gpgpu, + ): + m_debconf.DebconfCommunicator.side_effect = AttributeError + m_tmp.return_value = tmpdir + myCloud = mock.MagicMock() + version_none_cfg = { + "drivers": {"nvidia": {"license-accepted": True, "version": None}} + } + with pytest.raises(AttributeError): drivers.handle( - "ubuntu_drivers", self.cfg_accepted, myCloud, None, None + "ubuntu_drivers", version_none_cfg, myCloud, None, None ) - self.assertEqual( - [mock.call(["ubuntu-drivers-common"])], - myCloud.distro.install_packages.call_args_list, - ) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, - ) - self.assertIn( - "WARNING: the available version of ubuntu-drivers is" - " too old to perform requested driver installation", - self.logs.getvalue(), + assert ( + 0 == m_debconf.DebconfCommunicator.__enter__().command.call_count ) + assert 0 == m_subp.call_count + +@mock.patch(MPATH + "debconf") +@mock.patch(MPATH + "HAS_DEBCONF", True) +class TestUbuntuDriversWithVersion: + """With-version specific tests""" -# Sub-class TestUbuntuDrivers to run the same test cases, but with a version -class TestUbuntuDriversWithVersion(TestUbuntuDrivers): cfg_accepted = { "drivers": {"nvidia": {"license-accepted": True, "version": "123"}} } @@ -256,30 +320,76 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers): @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=False) - def test_version_none_uses_latest(self, m_which, m_subp, m_tmp): - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") - m_tmp.return_value = tdir + def test_version_none_uses_latest( + self, m_which, m_subp, m_tmp, m_debconf, tmpdir + ): + debconf_file = tmpdir.join("nvidia.template") + m_tmp.return_value = tmpdir myCloud = mock.MagicMock() version_none_cfg = { "drivers": {"nvidia": {"license-accepted": True, "version": None}} } drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, None, None) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(["ubuntu-drivers", "install", "--gpgpu", "nvidia"]), - ], - m_subp.call_args_list, + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers", "install", "--gpgpu", "nvidia"]), + ] == m_subp.call_args_list + + +@mock.patch(MPATH + "debconf") +class TestUbuntuDriversNotRun: + @mock.patch(MPATH + "HAS_DEBCONF", True) + @mock.patch(M_TMP_PATH) + @mock.patch(MPATH + "install_drivers") + def test_no_cfg_drivers_does_nothing( + self, + m_install_drivers, + m_tmp, + m_debconf, + tmpdir, + ): + m_tmp.return_value = tmpdir + m_log = mock.MagicMock() + myCloud = mock.MagicMock() + version_none_cfg = {} + drivers.handle( + "ubuntu_drivers", version_none_cfg, myCloud, m_log, None + ) + assert 0 == m_install_drivers.call_count + assert ( + mock.call( + "Skipping module named %s, no 'drivers' key in config", + "ubuntu_drivers", + ) + == m_log.debug.call_args_list[-1] ) - def test_specifying_a_version_doesnt_override_license_acceptance(self): - self._assert_inert_with_config( - { - "drivers": { - "nvidia": {"license-accepted": False, "version": "123"} - } - } + @mock.patch(MPATH + "HAS_DEBCONF", False) + @mock.patch(M_TMP_PATH) + @mock.patch(MPATH + "install_drivers") + def test_has_not_debconf_does_nothing( + self, + m_install_drivers, + m_tmp, + m_debconf, + tmpdir, + ): + m_tmp.return_value = tmpdir + m_log = mock.MagicMock() + myCloud = mock.MagicMock() + version_none_cfg = {"drivers": {"nvidia": {"license-accepted": True}}} + drivers.handle( + "ubuntu_drivers", version_none_cfg, myCloud, m_log, None + ) + assert 0 == m_install_drivers.call_count + assert ( + mock.call( + "Skipping module named %s, 'python3-debconf' is not installed", + "ubuntu_drivers", + ) + == m_log.warning.call_args_list[-1] ) diff --git a/tests/unittests/config/test_cc_update_etc_hosts.py b/tests/unittests/config/test_cc_update_etc_hosts.py index f7aafe462..d48656f7c 100644 --- a/tests/unittests/config/test_cc_update_etc_hosts.py +++ b/tests/unittests/config/test_cc_update_etc_hosts.py @@ -2,7 +2,6 @@ import logging import os -import re import shutil import pytest @@ -78,21 +77,35 @@ def test_write_etc_hosts_suse_template(self): class TestUpdateEtcHosts: @pytest.mark.parametrize( - "config, error_msg", + "config, expectation", [ + ({"manage_etc_hosts": True}, t_help.does_not_raise()), + ({"manage_etc_hosts": False}, t_help.does_not_raise()), + ({"manage_etc_hosts": "localhost"}, t_help.does_not_raise()), + ( + {"manage_etc_hosts": "template"}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: manage_etc_hosts: DEPRECATED. Value" + " ``template`` will be dropped after April 2027." + " Use ``true`` instead" + ), + ), + ), ( {"manage_etc_hosts": "templatey"}, - re.escape( - "manage_etc_hosts: 'templatey' is not one of" - " [True, False, 'template', 'localhost']" + pytest.raises( + SchemaValidationError, + match=( + "manage_etc_hosts: 'templatey' is not valid under any" + " of the given schemas" + ), ), ), ], ) @t_help.skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): - if error_msg is None: + def test_schema_validation(self, config, expectation): + with expectation: validate_cloudconfig_schema(config, get_schema(), strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index af8bdc302..12cdaa195 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -9,7 +9,12 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import ( + CiTestCase, + does_not_raise, + mock, + skipUnlessJsonSchema, +) MODPATH = "cloudinit.config.cc_users_groups" @@ -298,41 +303,190 @@ def test_users_ssh_redirect_user_and_no_default(self, m_user, m_group): class TestUsersGroupsSchema: @pytest.mark.parametrize( - "config, error_msg", + "config, expectation, has_errors", [ # Validate default settings not covered by examples - ({"groups": ["anygrp"]}, None), - ({"groups": "anygrp,anyothergroup"}, None), # DEPRECATED + ({"groups": ["anygrp"]}, does_not_raise(), None), + ( + {"groups": "anygrp,anyothergroup"}, + does_not_raise(), + None, + ), # DEPRECATED # Create anygrp with user1 as member - ({"groups": [{"anygrp": "user1"}]}, None), + ({"groups": [{"anygrp": "user1"}]}, does_not_raise(), None), # Create anygrp with user1 as member using object/string syntax - ({"groups": {"anygrp": "user1"}}, None), + ({"groups": {"anygrp": "user1"}}, does_not_raise(), None), # Create anygrp with user1 as member using object/list syntax - ({"groups": {"anygrp": ["user1"]}}, None), - ({"groups": [{"anygrp": ["user1", "user2"]}]}, None), + ({"groups": {"anygrp": ["user1"]}}, does_not_raise(), None), + ( + {"groups": [{"anygrp": ["user1", "user2"]}]}, + does_not_raise(), + None, + ), # Make default username "olddefault": DEPRECATED - ({"user": "olddefault"}, None), + ({"user": "olddefault"}, does_not_raise(), None), # Create multiple users, and include default user. DEPRECATED - ({"users": "oldstyle,default"}, None), - ({"users": ["default"]}, None), - ({"users": ["default", ["aaa", "bbb"]]}, None), - ({"users": ["foobar"]}, None), # no default user creation - ({"users": [{"name": "bbsw"}]}, None), - ({"groups": [{"yep": ["user1"]}]}, None), + ({"users": [{"name": "bbsw"}]}, does_not_raise(), None), + ( + {"users": [{"name": "bbsw", "garbage-key": None}]}, + pytest.raises( + SchemaValidationError, + match="is not valid under any of the given schemas", + ), + True, + ), + ( + {"groups": {"": "bbsw"}}, + pytest.raises( + SchemaValidationError, + match="does not match any of the regexes", + ), + True, + ), + ( + {"users": [{"name": "bbsw", "groups": ["anygrp"]}]}, + does_not_raise(), + None, + ), # user with a list of groups + ({"groups": [{"yep": ["user1"]}]}, does_not_raise(), None), + ({"users": "oldstyle,default"}, does_not_raise(), None), + ({"users": ["default"]}, does_not_raise(), None), + ({"users": ["default", ["aaa", "bbb"]]}, does_not_raise(), None), + # no default user creation + ({"users": ["foobar"]}, does_not_raise(), None), + ( + {"users": [{"name": "bbsw", "lock-passwd": True}]}, + pytest.raises( + SchemaValidationError, + match=( + "users.0.lock-passwd: DEPRECATED." + " Dropped after April 2027. Use ``lock_passwd``." + " Default: ``true``" + ), + ), + False, + ), + # users.groups supports comma-delimited str, list and object type + ( + {"users": [{"name": "bbsw", "groups": "adm, sudo"}]}, + does_not_raise(), + None, + ), + ( + { + "users": [ + {"name": "bbsw", "groups": {"adm": None, "sudo": None}} + ] + }, + pytest.raises( + SchemaValidationError, + match=( + "Cloud config schema deprecations: users.0.groups.adm:" + " DEPRECATED. When providing an object for" + " users.groups the ```` keys are the" + " groups to add this user to," + ), + ), + False, + ), + ({"groups": [{"yep": ["user1"]}]}, does_not_raise(), None), ( {"user": ["no_list_allowed"]}, - re.escape("user: ['no_list_allowed'] is not valid "), + pytest.raises( + SchemaValidationError, + match=re.escape("user: ['no_list_allowed'] is not valid "), + ), + True, ), ( {"groups": {"anygrp": 1}}, - "groups.anygrp: 1 is not of type 'string', 'array'", + pytest.raises( + SchemaValidationError, + match="groups.anygrp: 1 is not of type 'string', 'array'", + ), + True, + ), + ( + { + "users": [{"inactive": True, "name": "cloudy"}], + }, + pytest.raises( + SchemaValidationError, + match="errors: users.0: {'inactive': True", + ), + True, + ), + ( + { + "users": [ + { + "expiredate": "2038-01-19", + "groups": "users", + "name": "foobar", + } + ] + }, + does_not_raise(), + None, + ), + ( + {"user": {"name": "aciba", "groups": {"sbuild": None}}}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: user.groups.sbuild: DEPRECATED. " + "When providing an object for users.groups the " + "```` keys are the groups to add this " + "user to" + ), + ), + False, + ), + ( + {"user": {"name": "mynewdefault", "sudo": False}}, + pytest.raises( + SchemaValidationError, + match=( + "deprecations: user.sudo: DEPRECATED. The value" + " ``false`` will be dropped after April 2027." + " Use ``null`` or no ``sudo`` key instead." + ), + ), + False, + ), + ( + {"user": {"name": "mynewdefault", "sudo": None}}, + does_not_raise(), + None, + ), + ( + {"users": [{"name": "a", "uid": "1743"}]}, + pytest.raises( + SchemaValidationError, + match=( + "users.0.uid: DEPRECATED. The use of ``string`` type" + " will be dropped after April 2027. Use an ``integer``" + " instead." + ), + ), + False, + ), + ( + {"users": [{"name": "a", "expiredate": "2038,1,19"}]}, + pytest.raises( + SchemaValidationError, + match=( + "users.0: {'name': 'a', 'expiredate': '2038,1,19'}" + " is not valid under any of the given schemas" + ), + ), + True, ), ], ) @skipUnlessJsonSchema() - def test_schema_validation(self, config, error_msg): - if error_msg is None: + def test_schema_validation(self, config, expectation, has_errors): + with expectation as exc_info: validate_cloudconfig_schema(config, get_schema(), strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, get_schema(), strict=True) + if has_errors is not None: + assert has_errors == exc_info.value.has_errors() diff --git a/tests/unittests/config/test_cc_wireguard.py b/tests/unittests/config/test_cc_wireguard.py new file mode 100644 index 000000000..59a5223b1 --- /dev/null +++ b/tests/unittests/config/test_cc_wireguard.py @@ -0,0 +1,266 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import pytest + +from cloudinit import subp, util +from cloudinit.config import cc_wireguard +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema + +NL = "\n" +# Module path used in mocks +MPATH = "cloudinit.config.cc_wireguard" +MIN_KERNEL_VERSION = (5, 6) + + +class FakeCloud(object): + def __init__(self, distro): + self.distro = distro + + +class TestWireGuard(CiTestCase): + + with_logs = True + allowed_subp = [CiTestCase.SUBP_SHELL_TRUE] + + def setUp(self): + super(TestWireGuard, self).setUp() + self.tmp = self.tmp_dir() + + def test_readiness_probe_schema_non_string_values(self): + """ValueError raised for any values expected as string type.""" + wg_readinessprobes = [1, ["not-a-valid-command"]] + errors = [ + "Expected a string for readinessprobe at 0. Found 1", + "Expected a string for readinessprobe at 1." + " Found ['not-a-valid-command']", + ] + with self.assertRaises(ValueError) as context_mgr: + cc_wireguard.readinessprobe_command_validation(wg_readinessprobes) + error_msg = str(context_mgr.exception) + for error in errors: + self.assertIn(error, error_msg) + + def test_suppl_schema_error_on_missing_keys(self): + """ValueError raised reporting any missing required keys""" + cfg = {} + match = ( + f"Invalid wireguard interface configuration:{NL}" + "Missing required wg:interfaces keys: config_path, content, name" + ) + with self.assertRaisesRegex(ValueError, match): + cc_wireguard.supplemental_schema_validation(cfg) + + def test_suppl_schema_error_on_non_string_values(self): + """ValueError raised for any values expected as string type.""" + cfg = {"name": 1, "config_path": 2, "content": 3} + errors = [ + "Expected a string for wg:interfaces:config_path. Found 2", + "Expected a string for wg:interfaces:content. Found 3", + "Expected a string for wg:interfaces:name. Found 1", + ] + with self.assertRaises(ValueError) as context_mgr: + cc_wireguard.supplemental_schema_validation(cfg) + error_msg = str(context_mgr.exception) + for error in errors: + self.assertIn(error, error_msg) + + def test_write_config_failed(self): + """Errors when writing config are raised.""" + wg_int = {"name": "wg0", "config_path": "/no/valid/path"} + + with self.assertRaises(RuntimeError) as context_mgr: + cc_wireguard.write_config(wg_int) + self.assertIn( + "Failure writing Wireguard configuration file /no/valid/path:\n", + str(context_mgr.exception), + ) + + @mock.patch("%s.subp.subp" % MPATH) + def test_readiness_probe_invalid_command(self, m_subp): + """Errors when executing readinessprobes are raised.""" + wg_readinessprobes = ["not-a-valid-command"] + + def fake_subp(cmd, capture=None, shell=None): + fail_cmds = ["not-a-valid-command"] + if cmd in fail_cmds and capture and shell: + raise subp.ProcessExecutionError( + "not-a-valid-command: command not found" + ) + + m_subp.side_effect = fake_subp + + with self.assertRaises(RuntimeError) as context_mgr: + cc_wireguard.readinessprobe(wg_readinessprobes) + self.assertIn( + "Failed running readinessprobe command:\n" + "not-a-valid-command: Unexpected error while" + " running command.\n" + "Command: -\nExit code: -\nReason: -\n" + "Stdout: not-a-valid-command: command not found\nStderr: -", + str(context_mgr.exception), + ) + + @mock.patch("%s.subp.subp" % MPATH) + def test_enable_wg_on_error(self, m_subp): + """Errors when enabling wireguard interfaces are raised.""" + wg_int = {"name": "wg0"} + distro = mock.MagicMock() # No errors raised + distro.manage_service.side_effect = subp.ProcessExecutionError( + "systemctl start wg-quik@wg0 failed: exit code 1" + ) + mycloud = FakeCloud(distro) + with self.assertRaises(RuntimeError) as context_mgr: + cc_wireguard.enable_wg(wg_int, mycloud) + self.assertEqual( + "Failed enabling/starting Wireguard interface(s):\n" + "Unexpected error while running command.\n" + "Command: -\nExit code: -\nReason: -\n" + "Stdout: systemctl start wg-quik@wg0 failed: exit code 1\n" + "Stderr: -", + str(context_mgr.exception), + ) + + @mock.patch("%s.subp.which" % MPATH) + def test_maybe_install_wg_packages_noop_when_wg_tools_present( + self, m_which + ): + """Do nothing if wireguard-tools already exists.""" + m_which.return_value = "/usr/bin/wg" # already installed + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + "Some apt error" + ) + cc_wireguard.maybe_install_wireguard_packages(cloud=FakeCloud(distro)) + + @mock.patch("%s.subp.which" % MPATH) + def test_maybe_install_wf_tools_raises_update_errors(self, m_which): + """maybe_install_wireguard_packages logs and raises + apt update errors.""" + m_which.return_value = None + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + "Some apt error" + ) + with self.assertRaises(RuntimeError) as context_manager: + cc_wireguard.maybe_install_wireguard_packages( + cloud=FakeCloud(distro) + ) + self.assertEqual("Some apt error", str(context_manager.exception)) + self.assertIn("Package update failed\nTraceback", self.logs.getvalue()) + + @mock.patch("%s.subp.which" % MPATH) + def test_maybe_install_wg_raises_install_errors(self, m_which): + """maybe_install_wireguard_packages logs and raises package + install errors.""" + m_which.return_value = None + distro = mock.MagicMock() + distro.update_package_sources.return_value = None + distro.install_packages.side_effect = RuntimeError( + "Some install error" + ) + with self.assertRaises(RuntimeError) as context_manager: + cc_wireguard.maybe_install_wireguard_packages( + cloud=FakeCloud(distro) + ) + self.assertEqual("Some install error", str(context_manager.exception)) + self.assertIn( + "Failed to install wireguard-tools\n", self.logs.getvalue() + ) + + @mock.patch("%s.subp.subp" % MPATH) + def test_load_wg_module_failed(self, m_subp): + """load_wireguard_kernel_module logs and raises + kernel modules loading error.""" + m_subp.side_effect = subp.ProcessExecutionError( + "Some kernel module load error" + ) + with self.assertRaises(subp.ProcessExecutionError) as context_manager: + cc_wireguard.load_wireguard_kernel_module() + self.assertEqual( + "Unexpected error while running command.\n" + "Command: -\nExit code: -\nReason: -\n" + "Stdout: Some kernel module load error\n" + "Stderr: -", + str(context_manager.exception), + ) + self.assertIn( + "WARNING: Could not load wireguard module:\n", self.logs.getvalue() + ) + + @mock.patch("%s.subp.which" % MPATH) + def test_maybe_install_wg_packages_happy_path(self, m_which): + """maybe_install_wireguard_packages installs wireguard-tools.""" + packages = ["wireguard-tools"] + + if util.kernel_version() < MIN_KERNEL_VERSION: + packages.append("wireguard") + + m_which.return_value = None + distro = mock.MagicMock() # No errors raised + cc_wireguard.maybe_install_wireguard_packages(cloud=FakeCloud(distro)) + distro.update_package_sources.assert_called_once_with() + distro.install_packages.assert_called_once_with(packages) + + @mock.patch("%s.maybe_install_wireguard_packages" % MPATH) + def test_handle_no_config(self, m_maybe_install_wireguard_packages): + """When no wireguard configuration is provided, nothing happens.""" + cfg = {} + cc_wireguard.handle( + "wg", cfg=cfg, cloud=None, log=self.logger, args=None + ) + self.assertIn( + "DEBUG: Skipping module named wg, no 'wireguard'" + " configuration found", + self.logs.getvalue(), + ) + self.assertEqual(m_maybe_install_wireguard_packages.call_count, 0) + + def test_readiness_probe_with_non_string_values(self): + """ValueError raised for any values expected as string type.""" + cfg = [1, 2] + errors = [ + "Expected a string for readinessprobe at 0. Found 1", + "Expected a string for readinessprobe at 1. Found 2", + ] + with self.assertRaises(ValueError) as context_manager: + cc_wireguard.readinessprobe_command_validation(cfg) + error_msg = str(context_manager.exception) + for error in errors: + self.assertIn(error, error_msg) + + +class TestWireguardSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + # Valid schemas + ( + { + "wireguard": { + "interfaces": [ + { + "name": "wg0", + "config_path": "/etc/wireguard/wg0.conf", + "content": "test", + } + ] + } + }, + None, + ), + ], + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + if error_msg is not None: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + validate_cloudconfig_schema(config, get_schema(), strict=True) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/config/test_cc_write_files.py b/tests/unittests/config/test_cc_write_files.py index 01c920e8b..a9a402653 100644 --- a/tests/unittests/config/test_cc_write_files.py +++ b/tests/unittests/config/test_cc_write_files.py @@ -65,6 +65,7 @@ class TestWriteFiles(FilesystemMockingTestCase): with_logs = True + owner = "root:root" def setUp(self): super(TestWriteFiles, self).setUp() @@ -75,7 +76,11 @@ def test_simple(self): self.patchUtils(self.tmp) expected = "hello world\n" filename = "/tmp/my.file" - write_files("test_simple", [{"content": expected, "path": filename}]) + write_files( + "test_simple", + [{"content": expected, "path": filename}], + self.owner, + ) self.assertEqual(util.load_file(filename), expected) def test_append(self): @@ -88,13 +93,14 @@ def test_append(self): write_files( "test_append", [{"content": added, "path": filename, "append": "true"}], + self.owner, ) self.assertEqual(util.load_file(filename), expected) def test_yaml_binary(self): self.patchUtils(self.tmp) data = util.load_yaml(YAML_TEXT) - write_files("testname", data["write_files"]) + write_files("testname", data["write_files"], self.owner) for path, content in YAML_CONTENT_EXPECTED.items(): self.assertEqual(util.load_file(path), content) @@ -128,7 +134,7 @@ def test_all_decodings(self): files.append(cur) expected.append((cur["path"], data)) - write_files("test_decoding", files) + write_files("test_decoding", files, self.owner) for path, content in expected: self.assertEqual(util.load_file(path, decode=False), content) diff --git a/tests/unittests/config/test_cc_yum_add_repo.py b/tests/unittests/config/test_cc_yum_add_repo.py index d6de2ec21..6edd21f4c 100644 --- a/tests/unittests/config/test_cc_yum_add_repo.py +++ b/tests/unittests/config/test_cc_yum_add_repo.py @@ -60,12 +60,13 @@ def test_write_config(self): }, } self.patchUtils(self.tmp) + self.patchOS(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, LOG, []) - contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") + contents = util.load_file("/etc/yum.repos.d/epel-testing.repo") parser = configparser.ConfigParser() parser.read_string(contents) expected = { - "epel_testing": { + "epel-testing": { "name": "Extra Packages for Enterprise Linux 5 - Testing", "failovermethod": "priority", "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL", @@ -101,11 +102,11 @@ def test_write_config_array(self): } self.patchUtils(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, LOG, []) - contents = util.load_file("/etc/yum.repos.d/puppetlabs_products.repo") + contents = util.load_file("/etc/yum.repos.d/puppetlabs-products.repo") parser = configparser.ConfigParser() parser.read_string(contents) expected = { - "puppetlabs_products": { + "puppetlabs-products": { "name": "Puppet Labs Products El 6 - $basearch", "baseurl": "http://yum.puppetlabs.com/el/6/products/$basearch", "gpgkey": ( @@ -150,6 +151,24 @@ class TestAddYumRepoSchema: {"yum_repos": {"My Repo": {"enabled": "nope"}}}, "yum_repos.My Repo.enabled: 'nope' is not of type 'boolean'", ), + ( + { + "yum_repos": { + "hotwheels repo": {"": "config option requires a name"} + } + }, + "does not match any of the regexes", + ), + ( + { + "yum_repos": { + "matchbox repo": { + "$$$$$": "config option requires a valid name" + } + } + }, + "does not match any of the regexes", + ), ], ) @helpers.skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_modules.py b/tests/unittests/config/test_modules.py new file mode 100644 index 000000000..bc105064b --- /dev/null +++ b/tests/unittests/config/test_modules.py @@ -0,0 +1,174 @@ +# This file is part of cloud-init. See LICENSE file for license information. + + +import importlib +import logging +from pathlib import Path +from typing import List + +import pytest + +from cloudinit import util +from cloudinit.config.modules import ModuleDetails, Modules, _is_active +from cloudinit.config.schema import MetaSchema +from cloudinit.distros import ALL_DISTROS +from cloudinit.settings import FREQUENCIES +from tests.unittests.helpers import cloud_init_project_dir, mock + +M_PATH = "cloudinit.config.modules." + + +def get_module_names() -> List[str]: + """Return list of module names in cloudinit/config""" + files = list( + Path(cloud_init_project_dir("cloudinit/config/")).glob("cc_*.py") + ) + + return [mod.stem for mod in files] + + +def get_modules(): + examples = [] + for mod_name in get_module_names(): + module = importlib.import_module(f"cloudinit.config.{mod_name}") + for i, example in enumerate(module.meta.get("examples", [])): + examples.append( + pytest.param( + mod_name, module, example, id=f"{mod_name}_example_{i}" + ) + ) + return examples + + +class TestModules: + @pytest.mark.parametrize("frequency", FREQUENCIES) + @pytest.mark.parametrize( + "activate_by_schema_keys, cfg, active", + [ + (None, {}, True), + (None, {"module_name": {"x": "y"}}, True), + ([], {"module_name": {"x": "y"}}, True), + (["module_name"], {"module_name": {"x": "y"}}, True), + ( + ["module_name", "other_module"], + {"module_name": {"x": "y"}}, + True, + ), + (["module_name"], {"other_module": {"x": "y"}}, False), + ( + ["x"], + {"module_name": {"x": "y"}, "other_module": {"x": "y"}}, + False, + ), + ], + ) + def test__is_inapplicable( + self, activate_by_schema_keys, cfg, active, frequency + ): + module = mock.Mock() + module.meta = MetaSchema( + name="module_name", + id="cc_module_name", + title="title", + description="description", + distros=[ALL_DISTROS], + examples=["example_0", "example_1"], + frequency=frequency, + ) + if activate_by_schema_keys is not None: + module.meta["activate_by_schema_keys"] = activate_by_schema_keys + module_details = ModuleDetails( + module=module, + name="name", + frequency=frequency, + run_args=[], + ) + assert active == _is_active(module_details, cfg) + + @pytest.mark.parametrize("mod_name, module, example", get_modules()) + def test__is_inapplicable_examples(self, mod_name, module, example): + module_details = ModuleDetails( + module=module, + name=mod_name, + frequency=["always"], + run_args=[], + ) + assert True is _is_active(module_details, util.load_yaml(example)) + + @pytest.mark.parametrize("frequency", FREQUENCIES) + @pytest.mark.parametrize("active", [True, False]) + def test_run_section(self, frequency, active, caplog, mocker): + mocker.patch(M_PATH + "_is_active", return_value=active) + + mods = Modules( + init=mock.Mock(), cfg_files=mock.Mock(), reporter=mock.Mock() + ) + mods._cached_cfg = {} + raw_name = "my_module" + module = mock.Mock() + module.meta = MetaSchema( + name=raw_name, + id=f"cc_{raw_name}", + title="title", + description="description", + distros=[ALL_DISTROS], + examples=["example_0", "example_1"], + frequency=frequency, + ) + module_details = ModuleDetails( + module=module, + name=raw_name, + frequency=frequency, + run_args=[""], + ) + mocker.patch.object( + mods, + "_fixup_modules", + return_value=[module_details], + ) + m_run_modules = mocker.patch.object(mods, "_run_modules") + + assert mods.run_section("not_matter") + if active: + assert [ + mock.call([list(module_details)]) + ] == m_run_modules.call_args_list + assert not caplog.text + else: + assert [mock.call([])] == m_run_modules.call_args_list + assert ( + logging.INFO, + ( + f"Skipping modules '{raw_name}' because no applicable" + " config is provided." + ), + ) == caplog.record_tuples[-1][1:] + + @pytest.mark.parametrize("mod_name, module, example", get_modules()) + def test_run_section_examples( + self, mod_name, module, example, caplog, mocker + ): + mods = Modules( + init=mock.Mock(), cfg_files=mock.Mock(), reporter=mock.Mock() + ) + cfg = util.load_yaml(example) + cfg["unverified_modules"] = [mod_name] # Force to run unverified mod + mods._cached_cfg = cfg + module_details = ModuleDetails( + module=module, + name=mod_name, + frequency=["always"], + run_args=[], + ) + mocker.patch.object( + mods, + "_fixup_modules", + return_value=[module_details], + ) + mocker.patch.object(module, "handle") + m_run_modules = mocker.patch.object(mods, "_run_modules") + assert mods.run_section("not_matter") + assert [ + mock.call([list(module_details)]) + ] == m_run_modules.call_args_list + assert "Skipping" not in caplog.text diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 1fa91ad89..a401ffd46 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -7,26 +7,31 @@ import json import logging import os +import re import sys -from copy import copy, deepcopy +from collections import namedtuple +from copy import deepcopy from pathlib import Path from textwrap import dedent from types import ModuleType -from typing import List +from typing import List, Optional, Sequence, Set -import jsonschema import pytest +import responses +from cloudinit import stages from cloudinit.config.schema import ( CLOUD_CONFIG_HEADER, VERSIONED_USERDATA_SCHEMA_FILE, MetaSchema, + SchemaProblem, SchemaValidationError, annotated_cloudconfig_file, get_jsonschema_validator, get_meta_doc, get_schema, get_schema_dir, + handle_schema_args, load_doc, main, validate_cloudconfig_file, @@ -37,12 +42,19 @@ from cloudinit.safeyaml import load, load_with_marks from cloudinit.settings import FREQUENCIES from cloudinit.util import load_file, write_file +from tests.hypothesis import given +from tests.hypothesis_jsonschema import from_schema from tests.unittests.helpers import ( CiTestCase, cloud_init_project_dir, + does_not_raise, mock, + skipUnlessHypothesisJsonSchema, skipUnlessJsonSchema, ) +from tests.unittests.util import FakeDataSource + +M_PATH = "cloudinit.config.schema." def get_schemas() -> dict: @@ -80,7 +92,7 @@ def get_modules() -> List[ModuleType]: def get_module_variable(var_name) -> dict: """Inspect modules and get variable from module matching var_name""" - schemas = {} + schemas: dict = {} get_modules() for k, v in sys.modules.items(): path = Path(k) @@ -96,16 +108,6 @@ def get_module_variable(var_name) -> dict: class TestVersionedSchemas: - def _relative_ref_to_local_file_path(self, source_schema): - """Replace known relative ref URLs with full file path.""" - # jsonschema 2.6.0 doesn't support relative URLs in $refs (bionic) - full_path_schema = deepcopy(source_schema) - relative_ref = full_path_schema["oneOf"][0]["allOf"][1]["$ref"] - full_local_filepath = get_schema_dir() + relative_ref[1:] - file_ref = f"file://{full_local_filepath}" - full_path_schema["oneOf"][0]["allOf"][1]["$ref"] = file_ref - return full_path_schema - @pytest.mark.parametrize( "schema,error_msg", ( @@ -119,39 +121,30 @@ def _relative_ref_to_local_file_path(self, source_schema): def test_versioned_cloud_config_schema_is_valid_json( self, schema, error_msg ): + schema_dir = get_schema_dir() version_schemafile = os.path.join( - get_schema_dir(), VERSIONED_USERDATA_SCHEMA_FILE + schema_dir, VERSIONED_USERDATA_SCHEMA_FILE + ) + # Point to local schema files avoid JSON resolver trying to pull the + # reference from our upstream raw file in github. + version_schema = json.loads( + re.sub( + r"https:\/\/raw.githubusercontent.com\/canonical\/" + r"cloud-init\/main\/cloudinit\/config\/schemas\/", + f"file://{schema_dir}/", + load_file(version_schemafile), + ) ) - version_schema = json.loads(load_file(version_schemafile)) - # To avoid JSON resolver trying to pull the reference from our - # upstream raw file in github. - version_schema["$id"] = f"file://{version_schemafile}" if error_msg: with pytest.raises(SchemaValidationError) as context_mgr: - try: - validate_cloudconfig_schema( - schema, schema=version_schema, strict=True - ) - except jsonschema.exceptions.RefResolutionError: - full_path_schema = self._relative_ref_to_local_file_path( - version_schema - ) - validate_cloudconfig_schema( - schema, schema=full_path_schema, strict=True - ) - assert error_msg in str(context_mgr.value) - else: - try: validate_cloudconfig_schema( schema, schema=version_schema, strict=True ) - except jsonschema.exceptions.RefResolutionError: - full_path_schema = self._relative_ref_to_local_file_path( - version_schema - ) - validate_cloudconfig_schema( - schema, schema=full_path_schema, strict=True - ) + assert error_msg in str(context_mgr.value) + else: + validate_cloudconfig_schema( + schema, schema=version_schema, strict=True + ) class TestGetSchema: @@ -172,14 +165,15 @@ def test_get_schema_coalesces_known_schema(self): assert ["$defs", "$schema", "allOf"] == sorted(list(schema.keys())) # New style schema should be defined in static schema file in $defs expected_subschema_defs = [ + {"$ref": "#/$defs/cc_ansible"}, {"$ref": "#/$defs/cc_apk_configure"}, {"$ref": "#/$defs/cc_apt_configure"}, {"$ref": "#/$defs/cc_apt_pipelining"}, + {"$ref": "#/$defs/cc_ubuntu_autoinstall"}, {"$ref": "#/$defs/cc_bootcmd"}, {"$ref": "#/$defs/cc_byobu"}, {"$ref": "#/$defs/cc_ca_certs"}, {"$ref": "#/$defs/cc_chef"}, - {"$ref": "#/$defs/cc_debug"}, {"$ref": "#/$defs/cc_disable_ec2_metadata"}, {"$ref": "#/$defs/cc_disk_setup"}, {"$ref": "#/$defs/cc_fan"}, @@ -221,9 +215,11 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_update_etc_hosts"}, {"$ref": "#/$defs/cc_update_hostname"}, {"$ref": "#/$defs/cc_users_groups"}, + {"$ref": "#/$defs/cc_wireguard"}, {"$ref": "#/$defs/cc_write_files"}, {"$ref": "#/$defs/cc_yum_add_repo"}, {"$ref": "#/$defs/cc_zypper_add_repo"}, + {"$ref": "#/$defs/reporting_config"}, ] found_subschema_defs = [] legacy_schema_keys = [] @@ -258,10 +254,12 @@ class SchemaValidationErrorTest(CiTestCase): def test_schema_validation_error_expects_schema_errors(self): """SchemaValidationError is initialized from schema_errors.""" - errors = ( - ("key.path", 'unexpected key "junk"'), - ("key2.path", '"-123" is not a valid "hostname" format'), - ) + errors = [ + SchemaProblem("key.path", 'unexpected key "junk"'), + SchemaProblem( + "key2.path", '"-123" is not a valid "hostname" format' + ), + ] exception = SchemaValidationError(schema_errors=errors) self.assertIsInstance(exception, Exception) self.assertEqual(exception.schema_errors, errors) @@ -304,7 +302,7 @@ def test_validateconfig_schema_non_strict_emits_warnings(self, caplog): assert "cloudinit.config.schema" == module assert logging.WARNING == log_level assert ( - "Invalid cloud-config provided: \np1: -1 is not of type 'string'" + "Invalid cloud-config provided:\np1: -1 is not of type 'string'" == log_msg ) @@ -392,6 +390,201 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( in caplog.text ) + @skipUnlessJsonSchema() + @pytest.mark.parametrize("log_deprecations", [True, False]) + @pytest.mark.parametrize( + "schema,config,expected_msg", + [ + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "a-b": { + "type": "string", + "deprecated": True, + "description": "", + }, + "a_b": {"type": "string", "description": "noop"}, + }, + }, + {"a-b": "asdf"}, + "Deprecated cloud-config provided:\na-b: DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "x": { + "oneOf": [ + {"type": "integer", "description": "noop"}, + { + "type": "string", + "deprecated": True, + "description": "", + }, + ] + }, + }, + }, + {"x": "+5"}, + "Deprecated cloud-config provided:\nx: DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "x": { + "allOf": [ + {"type": "string", "description": "noop"}, + { + "deprecated": True, + "description": "", + }, + ] + }, + }, + }, + {"x": "5"}, + "Deprecated cloud-config provided:\nx: DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "x": { + "anyOf": [ + {"type": "integer", "description": "noop"}, + { + "type": "string", + "deprecated": True, + "description": "", + }, + ] + }, + }, + }, + {"x": "5"}, + "Deprecated cloud-config provided:\nx: DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "x": { + "type": "string", + "deprecated": True, + "description": "", + }, + }, + }, + {"x": "+5"}, + "Deprecated cloud-config provided:\nx: DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "x": { + "type": "string", + "deprecated": False, + "description": "", + }, + }, + }, + {"x": "+5"}, + None, + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "$defs": { + "my_ref": { + "deprecated": True, + "description": "", + } + }, + "properties": { + "x": { + "allOf": [ + {"type": "string", "description": "noop"}, + {"$ref": "#/$defs/my_ref"}, + ] + }, + }, + }, + {"x": "+5"}, + "Deprecated cloud-config provided:\nx: DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "$defs": { + "my_ref": { + "deprecated": True, + } + }, + "properties": { + "x": { + "allOf": [ + { + "type": "string", + "description": "noop", + }, + {"$ref": "#/$defs/my_ref"}, + ] + }, + }, + }, + {"x": "+5"}, + "Deprecated cloud-config provided:\nx: DEPRECATED.", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "patternProperties": { + "^.+$": { + "minItems": 1, + "deprecated": True, + "description": "", + } + }, + }, + {"a-b": "asdf"}, + "Deprecated cloud-config provided:\na-b: DEPRECATED: ", + ), + pytest.param( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "patternProperties": { + "^.+$": { + "minItems": 1, + "deprecated": True, + } + }, + }, + {"a-b": "asdf"}, + "Deprecated cloud-config provided:\na-b: DEPRECATED.", + id="deprecated_pattern_property_without_description", + ), + ], + ) + def test_validateconfig_logs_deprecations( + self, schema, config, expected_msg, log_deprecations, caplog + ): + validate_cloudconfig_schema( + config, + schema, + strict_metaschema=True, + log_deprecations=log_deprecations, + ) + if expected_msg is None: + return + log_record = (M_PATH[:-1], logging.WARNING, expected_msg) + if log_deprecations: + assert log_record == caplog.record_tuples[-1] + else: + assert log_record not in caplog.record_tuples + class TestCloudConfigExamples: metas = get_metas() @@ -439,6 +632,7 @@ def test_validateconfig_schema_of_example(self, schema_id, example): validate_cloudconfig_schema(config_load, schema, strict=True) +@pytest.mark.usefixtures("fake_filesystem") class TestValidateCloudConfigFile: """Tests for validate_cloudconfig_file.""" @@ -499,7 +693,7 @@ def test_validateconfig_file_error_on_non_yaml_parser_error( @skipUnlessJsonSchema() @pytest.mark.parametrize("annotate", (True, False)) - def test_validateconfig_file_sctrictly_validates_schema( + def test_validateconfig_file_strictly_validates_schema( self, annotate, tmpdir ): """validate_cloudconfig_file raises errors on invalid schema.""" @@ -512,6 +706,78 @@ def test_validateconfig_file_sctrictly_validates_schema( with pytest.raises(SchemaValidationError, match=error_msg): validate_cloudconfig_file(config_file.strpath, schema, annotate) + @skipUnlessJsonSchema() + @responses.activate + @pytest.mark.parametrize("annotate", (True, False)) + @mock.patch("cloudinit.url_helper.time.sleep") + @mock.patch(M_PATH + "os.getuid", return_value=0) + def test_validateconfig_file_include_validates_schema( + self, m_getuid, m_sleep, annotate, mocker + ): + """validate_cloudconfig_file raises errors on invalid schema + when user-data uses `#include`.""" + schema = {"properties": {"p1": {"type": "string", "format": "string"}}} + included_data = "#cloud-config\np1: -1" + included_url = "http://asdf/user-data" + blob = f"#include {included_url}" + responses.add(responses.GET, included_url, included_data) + + ci = stages.Init() + ci.datasource = FakeDataSource(blob) + mocker.patch(M_PATH + "Init", return_value=ci) + + error_msg = ( + "Cloud config schema errors: p1: -1 is not of type 'string'" + ) + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_file(None, schema, annotate) + + @skipUnlessJsonSchema() + @responses.activate + @pytest.mark.parametrize("annotate", (True, False)) + @mock.patch("cloudinit.url_helper.time.sleep") + @mock.patch(M_PATH + "os.getuid", return_value=0) + def test_validateconfig_file_include_success( + self, m_getuid, m_sleep, annotate, mocker + ): + """validate_cloudconfig_file raises errors on invalid schema + when user-data uses `#include`.""" + schema = {"properties": {"p1": {"type": "string", "format": "string"}}} + included_data = "#cloud-config\np1: asdf" + included_url = "http://asdf/user-data" + blob = f"#include {included_url}" + responses.add(responses.GET, included_url, included_data) + + ci = stages.Init() + ci.datasource = FakeDataSource(blob) + mocker.patch(M_PATH + "Init", return_value=ci) + + validate_cloudconfig_file(None, schema, annotate) + + @skipUnlessJsonSchema() + @pytest.mark.parametrize("annotate", (True, False)) + @mock.patch("cloudinit.url_helper.time.sleep") + @mock.patch(M_PATH + "os.getuid", return_value=0) + def test_validateconfig_file_no_cloud_cfg( + self, m_getuid, m_sleep, annotate, capsys, mocker + ): + """validate_cloudconfig_file does noop with empty user-data.""" + schema = {"properties": {"p1": {"type": "string", "format": "string"}}} + blob = "" + + ci = stages.Init() + ci.datasource = FakeDataSource(blob) + mocker.patch(M_PATH + "Init", return_value=ci) + + with pytest.raises( + SchemaValidationError, + match=re.escape( + "Cloud config schema errors: format-l1.c1: File None needs" + ' to begin with "#cloud-config"' + ), + ): + validate_cloudconfig_file(None, schema, annotate) + class TestSchemaDocMarkdown: """Tests for get_meta_doc.""" @@ -532,14 +798,22 @@ class TestSchemaDocMarkdown: "frequency": "frequency", "distros": ["debian", "rhel"], "examples": [ - 'ex1:\n [don\'t, expand, "this"]', - "ex2: true", + 'prop1:\n [don\'t, expand, "this"]', + "prop2: true", ], } - def test_get_meta_doc_returns_restructured_text(self): + @pytest.mark.parametrize( + "meta_update", + [ + None, + {"activate_by_schema_keys": None}, + {"activate_by_schema_keys": []}, + ], + ) + def test_get_meta_doc_returns_restructured_text(self, meta_update): """get_meta_doc returns restructured text for a cloudinit schema.""" - full_schema = copy(self.required_schema) + full_schema = deepcopy(self.required_schema) full_schema.update( { "properties": { @@ -551,8 +825,62 @@ def test_get_meta_doc_returns_restructured_text(self): } } ) + meta = deepcopy(self.meta) + if meta_update: + meta.update(meta_update) + + doc = get_meta_doc(meta, full_schema) + assert ( + dedent( + """ + name + ---- + **Summary:** title + + description + + **Internal name:** ``id`` + + **Module frequency:** frequency + + **Supported distros:** debian, rhel + + **Config schema**: + **prop1:** (array of integer) prop-description. + + **Examples**:: + + prop1: + [don't, expand, "this"] + # --- Example2 --- + prop2: true + """ + ) + == doc + ) + + def test_get_meta_doc_full_with_activate_by_schema_keys(self): + full_schema = deepcopy(self.required_schema) + full_schema.update( + { + "properties": { + "prop1": { + "type": "array", + "description": "prop-description", + "items": {"type": "string"}, + }, + "prop2": { + "type": "boolean", + "description": "prop2-description", + }, + }, + } + ) + + meta = deepcopy(self.meta) + meta["activate_by_schema_keys"] = ["prop1", "prop2"] - doc = get_meta_doc(self.meta, full_schema) + doc = get_meta_doc(meta, full_schema) assert ( dedent( """ @@ -568,15 +896,19 @@ def test_get_meta_doc_returns_restructured_text(self): **Supported distros:** debian, rhel + **Activate only on keys:** ``prop1``, ``prop2`` + **Config schema**: - **prop1:** (array of integer) prop-description + **prop1:** (array of string) prop-description. + + **prop2:** (boolean) prop2-description. **Examples**:: - ex1: + prop1: [don't, expand, "this"] # --- Example2 --- - ex2: true + prop2: true """ ) == doc @@ -587,6 +919,23 @@ def test_get_meta_doc_handles_multiple_types(self): schema = {"properties": {"prop1": {"type": ["string", "integer"]}}} assert "**prop1:** (string/integer)" in get_meta_doc(self.meta, schema) + @pytest.mark.parametrize("multi_key", ["oneOf", "anyOf"]) + def test_get_meta_doc_handles_multiple_types_recursive(self, multi_key): + """get_meta_doc delimits multiple property types with a '/'.""" + schema = { + "properties": { + "prop1": { + multi_key: [ + {"type": ["string", "null"]}, + {"type": "integer"}, + ] + } + } + } + assert "**prop1:** (string/null/integer)" in get_meta_doc( + self.meta, schema + ) + def test_references_are_flattened_in_schema_docs(self): """get_meta_doc flattens and renders full schema definitions.""" schema = { @@ -612,7 +961,7 @@ def test_references_are_flattened_in_schema_docs(self): """\ **prop1:** (string/object) Objects support the following keys: - **:** (array of string) List of cool strings + **:** (array of string) List of cool strings. """ ) in get_meta_doc(self.meta, schema) @@ -686,14 +1035,17 @@ def test_get_meta_doc_hidden_hides_specific_properties_from_docs( """ assert expected in get_meta_doc(self.meta, schema) - def test_get_meta_doc_handles_nested_oneof_property_types(self): + @pytest.mark.parametrize("multi_key", ["oneOf", "anyOf"]) + def test_get_meta_doc_handles_nested_multi_schema_property_types( + self, multi_key + ): """get_meta_doc describes array items oneOf declarations in type.""" schema = { "properties": { "prop1": { "type": "array", "items": { - "oneOf": [{"type": "string"}, {"type": "integer"}] + multi_key: [{"type": "string"}, {"type": "integer"}] }, } } @@ -702,14 +1054,15 @@ def test_get_meta_doc_handles_nested_oneof_property_types(self): self.meta, schema ) - def test_get_meta_doc_handles_types_as_list(self): + @pytest.mark.parametrize("multi_key", ["oneOf", "anyOf"]) + def test_get_meta_doc_handles_types_as_list(self, multi_key): """get_meta_doc renders types which have a list value.""" schema = { "properties": { "prop1": { "type": ["boolean", "array"], "items": { - "oneOf": [{"type": "string"}, {"type": "integer"}] + multi_key: [{"type": "string"}, {"type": "integer"}] }, } } @@ -737,7 +1090,7 @@ def test_get_meta_doc_handles_flattening_defs(self): def test_get_meta_doc_handles_string_examples(self): """get_meta_doc properly indented examples as a list of strings.""" - full_schema = copy(self.required_schema) + full_schema = deepcopy(self.required_schema) full_schema.update( { "examples": [ @@ -757,14 +1110,14 @@ def test_get_meta_doc_handles_string_examples(self): dedent( """ **Config schema**: - **prop1:** (array of integer) prop-description + **prop1:** (array of integer) prop-description. **Examples**:: - ex1: + prop1: [don't, expand, "this"] # --- Example2 --- - ex2: true + prop2: true """ ) in get_meta_doc(self.meta, full_schema) @@ -803,14 +1156,15 @@ def test_get_meta_doc_properly_parse_description(self): - option2 - option3 - The default value is option1 + The default value is option1. """ ) in get_meta_doc(self.meta, schema) ) - def test_get_meta_doc_raises_key_errors(self): + @pytest.mark.parametrize("key", meta.keys()) + def test_get_meta_doc_raises_key_errors(self, key): """get_meta_doc raises KeyErrors on missing keys.""" schema = { "properties": { @@ -822,18 +1176,51 @@ def test_get_meta_doc_raises_key_errors(self): } } } - for key in self.meta: - invalid_meta = copy(self.meta) - invalid_meta.pop(key) - with pytest.raises(KeyError) as context_mgr: - get_meta_doc(invalid_meta, schema) - assert key in str(context_mgr.value) + invalid_meta = deepcopy(self.meta) + invalid_meta.pop(key) + with pytest.raises( + KeyError, + match=f"Missing required keys in module meta: {{'{key}'}}", + ): + get_meta_doc(invalid_meta, schema) + + @pytest.mark.parametrize( + "key,expectation", + [ + ("activate_by_schema_keys", does_not_raise()), + ( + "additional_key", + pytest.raises( + KeyError, + match=( + "Additional unexpected keys found in module meta:" + " {'additional_key'}" + ), + ), + ), + ], + ) + def test_get_meta_doc_additional_keys(self, key, expectation): + schema = { + "properties": { + "prop1": { + "type": "array", + "items": { + "oneOf": [{"type": "string"}, {"type": "integer"}] + }, + } + } + } + invalid_meta = deepcopy(self.meta) + invalid_meta[key] = [] + with expectation: + get_meta_doc(invalid_meta, schema) def test_label_overrides_property_name(self): """get_meta_doc overrides property name with label.""" schema = { "properties": { - "prop1": { + "old_prop1": { "type": "string", "label": "label1", }, @@ -864,9 +1251,186 @@ def test_label_overrides_property_name(self): assert "**prop_no_label:** (string)" in meta_doc assert "Each object in **array_label** list" in meta_doc - assert "prop1" not in meta_doc + assert "old_prop1" not in meta_doc assert ".*" not in meta_doc + @pytest.mark.parametrize( + "schema,expected_doc", + [ + ( + { + "properties": { + "prop1": { + "type": ["string", "integer"], + "deprecated": True, + "description": "", + } + } + }, + "**prop1:** (string/integer) DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "prop1": { + "type": ["string", "integer"], + "description": "", + "deprecated": True, + }, + }, + }, + "**prop1:** (string/integer) DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "$defs": {"my_ref": {"deprecated": True}}, + "properties": { + "prop1": { + "allOf": [ + { + "type": ["string", "integer"], + "description": "", + }, + {"$ref": "#/$defs/my_ref"}, + ] + } + }, + }, + "**prop1:** (string/integer) DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "$defs": { + "my_ref": { + "deprecated": True, + "description": "", + } + }, + "properties": { + "prop1": { + "allOf": [ + {"type": ["string", "integer"]}, + {"$ref": "#/$defs/my_ref"}, + ] + } + }, + }, + "**prop1:** (string/integer) DEPRECATED: ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "prop1": { + "description": "", + "anyOf": [ + { + "type": ["string", "integer"], + "description": "", + "deprecated": True, + }, + ], + }, + }, + }, + "**prop1:** (UNDEFINED) . DEPRECATED: ", + "deprecated": True, + }, + { + "type": "number", + "description": "", + }, + ] + }, + }, + }, + "**prop1:** (number) . DEPRECATED:" + " ", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "prop1": { + "anyOf": [ + { + "type": ["string", "integer"], + "description": "", + "deprecated": True, + }, + { + "type": "string", + "enum": ["none", "unchanged", "os"], + "description": "", + }, + ] + }, + }, + }, + "**prop1:** (``none``/``unchanged``/``os``) ." + " DEPRECATED: .", + ), + ( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + "prop1": { + "anyOf": [ + { + "type": ["string", "integer"], + "description": "", + }, + { + "type": "string", + "enum": ["none", "unchanged", "os"], + "description": "_2", + }, + ] + }, + }, + }, + "**prop1:** (string/integer/``none``/``unchanged``/``os``)" + " . _2.\n", + ), + ( + { + "properties": { + "prop1": { + "description": "", + "type": "array", + "items": { + "type": "object", + "anyOf": [ + { + "properties": { + "sub_prop1": {"type": "string"}, + }, + }, + ], + }, + }, + }, + }, + "**prop1:** (array of object) .\n", + ), + ], + ) + def test_get_meta_doc_render_deprecated_info(self, schema, expected_doc): + assert expected_doc in get_meta_doc(self.meta, schema) + class TestAnnotatedCloudconfigFile: def test_annotated_cloudconfig_file_no_schema_errors(self): @@ -874,7 +1438,10 @@ def test_annotated_cloudconfig_file_no_schema_errors(self): content = b"ntp:\n pools: [ntp1.pools.com]\n" parse_cfg, schemamarks = load_with_marks(content) assert content == annotated_cloudconfig_file( - parse_cfg, content, schema_errors=[], schemamarks=schemamarks + parse_cfg, + content, + schemamarks=schemamarks, + schema_errors=[], ) def test_annotated_cloudconfig_file_with_non_dict_cloud_config(self): @@ -896,8 +1463,8 @@ def test_annotated_cloudconfig_file_with_non_dict_cloud_config(self): assert expected == annotated_cloudconfig_file( None, content, - schema_errors=[("", "None is not of type 'object'")], schemamarks={}, + schema_errors=[SchemaProblem("", "None is not of type 'object'")], ) def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self): @@ -926,12 +1493,15 @@ def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self): ) parsed_config, schemamarks = load_with_marks(content[13:]) schema_errors = [ - ("ntp", "Some type error"), - ("ntp.pools.0", "-99 is not a string"), - ("ntp.pools.1", "75 is not a string"), + SchemaProblem("ntp", "Some type error"), + SchemaProblem("ntp.pools.0", "-99 is not a string"), + SchemaProblem("ntp.pools.1", "75 is not a string"), ] assert expected == annotated_cloudconfig_file( - parsed_config, content, schema_errors, schemamarks=schemamarks + parsed_config, + content, + schemamarks=schemamarks, + schema_errors=schema_errors, ) def test_annotated_cloudconfig_file_annotates_separate_line_items(self): @@ -956,11 +1526,14 @@ def test_annotated_cloudconfig_file_annotates_separate_line_items(self): ) parsed_config, schemamarks = load_with_marks(content[13:]) schema_errors = [ - ("ntp.pools.0", "-99 is not a string"), - ("ntp.pools.1", "75 is not a string"), + SchemaProblem("ntp.pools.0", "-99 is not a string"), + SchemaProblem("ntp.pools.1", "75 is not a string"), ] assert expected in annotated_cloudconfig_file( - parsed_config, content, schema_errors, schemamarks=schemamarks + parsed_config, + content, + schemamarks=schemamarks, + schema_errors=schema_errors, ) @@ -1042,15 +1615,15 @@ def test_main_validates_config_file(self, tmpdir, capsys): out, _err = capsys.readouterr() assert "Valid cloud-config: {0}\n".format(myyaml) == out - @mock.patch("cloudinit.config.schema.read_cfg_paths") - @mock.patch("cloudinit.config.schema.os.getuid", return_value=0) + @mock.patch(M_PATH + "os.getuid", return_value=0) def test_main_validates_system_userdata( - self, m_getuid, m_read_cfg_paths, capsys, paths + self, m_getuid, capsys, mocker, paths ): """When --system is provided, main validates system userdata.""" - m_read_cfg_paths.return_value = paths - ud_file = paths.get_ipath_cur("userdata_raw") - write_file(ud_file, b"#cloud-config\nntp:") + m_init = mocker.patch(M_PATH + "Init") + m_init.return_value.paths.get_ipath = paths.get_ipath_cur + cloud_config_file = paths.get_ipath_cur("cloud_config") + write_file(cloud_config_file, b"#cloud-config\nntp:") myargs = ["mycmd", "--system"] with mock.patch("sys.argv", myargs): assert 0 == main(), "Expected 0 exit code" @@ -1143,3 +1716,126 @@ def test_valid_meta_for_every_module(self): assert "distros" in module.meta assert {module.meta["frequency"]}.issubset(FREQUENCIES) assert set(module.meta["distros"]).issubset(all_distros) + + +def remove_modules(schema, modules: Set[str]) -> dict: + indices_to_delete = set() + for module in set(modules): + for index, ref_dict in enumerate(schema["allOf"]): + if ref_dict["$ref"] == f"#/$defs/{module}": + indices_to_delete.add(index) + continue # module found + for index in indices_to_delete: + schema["allOf"].pop(index) + return schema + + +def remove_defs(schema, defs: Set[str]) -> dict: + defs_to_delete = set(schema["$defs"].keys()).intersection(set(defs)) + for key in defs_to_delete: + del schema["$defs"][key] + return schema + + +def clean_schema( + schema=None, + modules: Optional[Sequence[str]] = None, + defs: Optional[Sequence[str]] = None, +): + schema = deepcopy(schema or get_schema()) + if modules: + remove_modules(schema, set(modules)) + if defs: + remove_defs(schema, set(defs)) + return schema + + +@pytest.mark.hypothesis_slow +class TestSchemaFuzz: + + # Avoid https://github.com/Zac-HD/hypothesis-jsonschema/issues/97 + SCHEMA = clean_schema( + modules=["cc_users_groups"], + defs=["users_groups.groups_by_groupname", "users_groups.user"], + ) + + @skipUnlessHypothesisJsonSchema() + @given(from_schema(SCHEMA)) + def test_validate_full_schema(self, config): + try: + validate_cloudconfig_schema(config, strict=True) + except SchemaValidationError as ex: + if ex.has_errors(): + raise + + +class TestHandleSchemaArgs: + + Args = namedtuple("Args", "config_file docs system annotate") + + @pytest.mark.parametrize( + "annotate, expected_output", + [ + ( + True, + dedent( + """\ + #cloud-config + packages: + - htop + apt_update: true # D1 + apt_upgrade: true # D2 + apt_reboot_if_required: true # D3 + + # Deprecations: ------------- + # D1: DEPRECATED: Dropped after April 2027. Use ``package_update``. Default: ``false`` + # D2: DEPRECATED: Dropped after April 2027. Use ``package_upgrade``. Default: ``false`` + # D3: DEPRECATED: Dropped after April 2027. Use ``package_reboot_if_required``. Default: ``false`` + + + Valid cloud-config: {} + """ # noqa: E501 + ), + ), + ( + False, + dedent( + """\ + Cloud config schema deprecations: \ +apt_reboot_if_required: DEPRECATED: Dropped after April 2027. Use ``package_reboot_if_required``. Default: ``false``, \ +apt_update: DEPRECATED: Dropped after April 2027. Use ``package_update``. Default: ``false``, \ +apt_upgrade: DEPRECATED: Dropped after April 2027. Use ``package_upgrade``. Default: ``false`` + Valid cloud-config: {} + """ # noqa: E501 + ), + ), + ], + ) + def test_handle_schema_args_annotate_deprecated_config( + self, annotate, expected_output, caplog, capsys, tmpdir + ): + user_data_fn = tmpdir.join("user-data") + with open(user_data_fn, "w") as f: + f.write( + dedent( + """\ + #cloud-config + packages: + - htop + apt_update: true + apt_upgrade: true + apt_reboot_if_required: true + """ + ) + ) + args = self.Args( + config_file=str(user_data_fn), + annotate=annotate, + docs=None, + system=None, + ) + handle_schema_args("unused", args) + out, err = capsys.readouterr() + assert expected_output.format(user_data_fn) == out + assert not err + assert "deprec" not in caplog.text diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py new file mode 100644 index 000000000..1ab17e8b1 --- /dev/null +++ b/tests/unittests/conftest.py @@ -0,0 +1,67 @@ +import builtins +import glob +import os +from pathlib import Path + +import pytest + +from cloudinit import atomic_helper, util +from tests.unittests.helpers import retarget_many_wrapper + +FS_FUNCS = { + os.path: [ + ("isfile", 1), + ("exists", 1), + ("islink", 1), + ("isdir", 1), + ("lexists", 1), + ("relpath", 1), + ], + os: [ + ("listdir", 1), + ("mkdir", 1), + ("lstat", 1), + ("symlink", 2), + ("stat", 1), + ("scandir", 1), + ], + util: [ + ("write_file", 1), + ("append_file", 1), + ("load_file", 1), + ("ensure_dir", 1), + ("chmod", 1), + ("delete_dir_contents", 1), + ("del_file", 1), + ("sym_link", -1), + ("copy", -1), + ], + glob: [ + ("glob", 1), + ], + builtins: [ + ("open", 1), + ], + atomic_helper: [ + ("write_file", 1), + ], +} + + +@pytest.fixture +def fake_filesystem(mocker, tmpdir): + """Mocks fs functions to operate under `tmpdir`""" + for (mod, funcs) in FS_FUNCS.items(): + for f, nargs in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(str(tmpdir), nargs, func) + mocker.patch.object(mod, f, trap_func) + + +PYTEST_VERSION_TUPLE = tuple(map(int, pytest.__version__.split("."))) + +if PYTEST_VERSION_TUPLE < (3, 9, 0): + + @pytest.fixture + def tmp_path(tmpdir): + return Path(tmpdir) diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index ddb039bd6..edc152e1f 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -135,6 +135,42 @@ def test_create_groups_with_whitespace_string( ] self.assertEqual(m_subp.call_args_list, expected) + @mock.patch("cloudinit.distros.util.is_group", return_value=False) + def test_create_groups_with_dict_deprecated( + self, m_is_group, m_subp, m_is_snappy + ): + """users.groups supports a dict value, but emit deprecation log.""" + user = "foouser" + self.dist.create_user(user, groups={"group1": None, "group2": None}) + expected = [ + mock.call(["groupadd", "group1"]), + mock.call(["groupadd", "group2"]), + self._useradd2call([user, "--groups", "group1,group2", "-m"]), + mock.call(["passwd", "-l", user]), + ] + self.assertEqual(m_subp.call_args_list, expected) + self.assertIn( + "WARNING: DEPRECATED: The user foouser has a 'groups' config" + " value of type dict which is deprecated and will be removed in a" + " future version of cloud-init. Use a comma-delimited string or" + " array instead: group1,group2.", + self.logs.getvalue(), + ) + + @mock.patch("cloudinit.distros.util.is_group", return_value=False) + def test_create_groups_with_list(self, m_is_group, m_subp, m_is_snappy): + """users.groups supports a list value.""" + user = "foouser" + self.dist.create_user(user, groups=["group1", "group2"]) + expected = [ + mock.call(["groupadd", "group1"]), + mock.call(["groupadd", "group2"]), + self._useradd2call([user, "--groups", "group1,group2", "-m"]), + mock.call(["passwd", "-l", user]), + ] + self.assertEqual(m_subp.call_args_list, expected) + self.assertNotIn("WARNING: DEPRECATED: ", self.logs.getvalue()) + def test_explicit_sudo_false(self, m_subp, m_is_snappy): user = "foouser" self.dist.create_user(user, sudo=False) @@ -145,6 +181,24 @@ def test_explicit_sudo_false(self, m_subp, m_is_snappy): mock.call(["passwd", "-l", user]), ], ) + self.assertIn( + "WARNING: DEPRECATED: The user foouser has a 'sudo' config value" + " of 'false' which will be dropped after April 2027. Use 'null'" + " instead.", + self.logs.getvalue(), + ) + + def test_explicit_sudo_none(self, m_subp, m_is_snappy): + user = "foouser" + self.dist.create_user(user, sudo=None) + self.assertEqual( + m_subp.call_args_list, + [ + self._useradd2call([user, "-m"]), + mock.call(["passwd", "-l", user]), + ], + ) + self.assertNotIn("WARNING: DEPRECATED: ", self.logs.getvalue()) @mock.patch("cloudinit.ssh_util.setup_user_keys") def test_setup_ssh_authorized_keys_with_string( diff --git a/tests/unittests/distros/test_netconfig.py b/tests/unittests/distros/test_netconfig.py index a25be4815..6509f1de8 100644 --- a/tests/unittests/distros/test_netconfig.py +++ b/tests/unittests/distros/test_netconfig.py @@ -9,6 +9,7 @@ from cloudinit import distros, helpers, safeyaml, settings, subp, util from cloudinit.distros.parsers.sys_conf import SysConf +from cloudinit.net.activators import IfUpDownActivator from tests.unittests.helpers import FilesystemMockingTestCase, dir2dict BASE_NET_CFG = """ @@ -234,6 +235,38 @@ """ +V2_PASSTHROUGH_NET_CFG = { + "ethernets": { + "eth7": { + "addresses": ["192.168.1.5/24"], + "gateway4": "192.168.1.254", + "routes": [{"to": "default", "via": "10.0.4.1", "metric": 100}], + }, + }, + "version": 2, +} + + +V2_PASSTHROUGH_NET_CFG_OUTPUT = """\ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +network: + ethernets: + eth7: + addresses: + - 192.168.1.5/24 + gateway4: 192.168.1.254 + routes: + - metric: 100 + to: default + via: 10.0.4.1 + version: 2 +""" + + class WriteBuffer(object): def __init__(self): self.buffer = StringIO() @@ -252,12 +285,17 @@ def setUp(self): super(TestNetCfgDistroBase, self).setUp() self.add_patch("cloudinit.util.system_is_snappy", "m_snappy") - def _get_distro(self, dname, renderers=None): + def _get_distro(self, dname, renderers=None, activators=None): cls = distros.fetch(dname) cfg = settings.CFG_BUILTIN cfg["system_info"]["distro"] = dname + system_info_network_cfg = {} if renderers: - cfg["system_info"]["network"] = {"renderers": renderers} + system_info_network_cfg["renderers"] = renderers + if activators: + system_info_network_cfg["activators"] = activators + if system_info_network_cfg: + cfg["system_info"]["network"] = system_info_network_cfg paths = helpers.Paths({}) return cls(dname, cfg.get("system_info"), paths) @@ -371,7 +409,9 @@ def test_apply_network_config_freebsd_nameserver(self, ifaces_mac): class TestNetCfgDistroUbuntuEni(TestNetCfgDistroBase): def setUp(self): super(TestNetCfgDistroUbuntuEni, self).setUp() - self.distro = self._get_distro("ubuntu", renderers=["eni"]) + self.distro = self._get_distro( + "ubuntu", renderers=["eni"], activators=["eni"] + ) def eni_path(self): return "/etc/network/interfaces.d/50-cloud-init.cfg" @@ -398,6 +438,51 @@ def _apply_and_verify_eni( self.assertEqual(expected, results[cfgpath]) self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + def test_apply_network_config_and_bringup_filters_priority_eni_ub(self): + """Network activator search priority can be overridden from config.""" + expected_cfgs = { + self.eni_path(): V1_NET_CFG_OUTPUT, + } + # ub_distro.apply_network_config(V1_NET_CFG, False) + with mock.patch( + "cloudinit.net.activators.select_activator" + ) as select_activator: + select_activator.return_value = IfUpDownActivator + self._apply_and_verify_eni( + self.distro.apply_network_config, + V1_NET_CFG, + expected_cfgs=expected_cfgs.copy(), + bringup=True, + ) + # 2nd call to select_activator via distro.network_activator prop + assert IfUpDownActivator == self.distro.network_activator + self.assertEqual( + [mock.call(priority=["eni"])] * 2, select_activator.call_args_list + ) + + def test_apply_network_config_and_bringup_activator_defaults_ub(self): + """Network activator search priority defaults when unspecified.""" + expected_cfgs = { + self.eni_path(): V1_NET_CFG_OUTPUT, + } + # Don't set activators to see DEFAULT_PRIORITY + self.distro = self._get_distro("ubuntu", renderers=["eni"]) + with mock.patch( + "cloudinit.net.activators.select_activator" + ) as select_activator: + select_activator.return_value = IfUpDownActivator + self._apply_and_verify_eni( + self.distro.apply_network_config, + V1_NET_CFG, + expected_cfgs=expected_cfgs.copy(), + bringup=True, + ) + # 2nd call to select_activator via distro.network_activator prop + assert IfUpDownActivator == self.distro.network_activator + self.assertEqual( + [mock.call(priority=None)] * 2, select_activator.call_args_list + ) + def test_apply_network_config_eni_ub(self): expected_cfgs = { self.eni_path(): V1_NET_CFG_OUTPUT, @@ -419,6 +504,9 @@ def test_apply_network_config_ipv6_ub(self): class TestNetCfgDistroUbuntuNetplan(TestNetCfgDistroBase): + + with_logs = True + def setUp(self): super(TestNetCfgDistroUbuntuNetplan, self).setUp() self.distro = self._get_distro("ubuntu", renderers=["netplan"]) @@ -487,6 +575,22 @@ def test_apply_network_config_v2_passthrough_ub(self): expected_cfgs=expected_cfgs.copy(), ) + def test_apply_network_config_v2_full_passthrough_ub(self): + expected_cfgs = { + self.netplan_path(): V2_PASSTHROUGH_NET_CFG_OUTPUT, + } + # ub_distro.apply_network_config(V2_PASSTHROUGH_NET_CFG, False) + self._apply_and_verify_netplan( + self.distro.apply_network_config, + V2_PASSTHROUGH_NET_CFG, + expected_cfgs=expected_cfgs.copy(), + ) + self.assertIn("Passthrough netplan v2 config", self.logs.getvalue()) + self.assertIn( + "Selected renderer 'netplan' from priority list: ['netplan']", + self.logs.getvalue(), + ) + class TestNetCfgDistroRedhat(TestNetCfgDistroBase): def setUp(self): diff --git a/tests/unittests/distros/test_networking.py b/tests/unittests/distros/test_networking.py index f56b34ad9..6f7465c94 100644 --- a/tests/unittests/distros/test_networking.py +++ b/tests/unittests/distros/test_networking.py @@ -2,7 +2,6 @@ # /parametrize.html#parametrizing-conditional-raising import textwrap -from contextlib import ExitStack as does_not_raise from unittest import mock import pytest @@ -14,6 +13,7 @@ LinuxNetworking, Networking, ) +from tests.unittests.helpers import does_not_raise @pytest.fixture diff --git a/tests/unittests/distros/test_sysconfig.py b/tests/unittests/distros/test_sysconfig.py index d0979e17e..9c3a2018e 100644 --- a/tests/unittests/distros/test_sysconfig.py +++ b/tests/unittests/distros/test_sysconfig.py @@ -65,9 +65,7 @@ def test_parse_adjust(self): conf["IPV6TO4_ROUTING"] = "blah \tblah" contents2 = str(conf).strip() # Should be requoted due to whitespace - self.assertRegMatches( - contents2, r"IPV6TO4_ROUTING=[\']blah\s+blah[\']" - ) + self.assertRegex(contents2, r"IPV6TO4_ROUTING=[\']blah\s+blah[\']") def test_parse_no_adjust_shell(self): conf = SysConf("".splitlines()) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 67fed8c9c..31e0188cd 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -13,10 +13,12 @@ import unittest from contextlib import ExitStack, contextmanager from pathlib import Path +from typing import ClassVar, List, Union from unittest import mock from unittest.util import strclass import httpretty +import pytest import cloudinit from cloudinit import cloud, distros @@ -28,6 +30,7 @@ ) from cloudinit.sources import DataSourceNone from cloudinit.templater import JINJA_AVAILABLE +from tests.hypothesis_jsonschema import HAS_HYPOTHESIS_JSONSCHEMA _real_subp = subp.subp @@ -71,6 +74,13 @@ def wrapper(*args, **kwds): return wrapper +def random_string(length=8): + """return a random lowercase string with default length of 8""" + return "".join( + random.choice(string.ascii_lowercase) for _ in range(length) + ) + + class TestCase(unittest.TestCase): def reset_global_state(self): """Reset any global state to its original settings. @@ -85,9 +95,7 @@ def reset_global_state(self): In the future this should really be done with some registry that can then be cleaned in a more obvious way. """ - util.PROC_CMDLINE = None util._DNS_REDIRECT_IP = None - util._LSB_RELEASE = {} def setUp(self): super(TestCase, self).setUp() @@ -114,7 +122,7 @@ class CiTestCase(TestCase): # Subclass overrides for specific test behavior # Whether or not a unit test needs logfile setup with_logs = False - allowed_subp = False + allowed_subp: ClassVar[Union[List, bool]] = False SUBP_SHELL_TRUE = "shell=true" @contextmanager @@ -226,10 +234,7 @@ def tmp_cloud(self, distro, sys_cfg=None, metadata=None): @classmethod def random_string(cls, length=8): - """return a random lowercase string with default length of 8""" - return "".join( - random.choice(string.ascii_lowercase) for _ in range(length) - ) + return random_string(length) class ResourceUsingTestCase(CiTestCase): @@ -518,6 +523,13 @@ def skipIfJinja(): return skipIf(JINJA_AVAILABLE, "Jinja dependency present.") +def skipUnlessHypothesisJsonSchema(): + return skipIf( + not HAS_HYPOTHESIS_JSONSCHEMA, + "No python-hypothesis-jsonschema dependency present.", + ) + + # older versions of mock do not have the useful 'assert_not_called' if not hasattr(mock.Mock, "assert_not_called"): @@ -530,7 +542,7 @@ def __mock_assert_not_called(mmock): ) raise AssertionError(msg) - mock.Mock.assert_not_called = __mock_assert_not_called + mock.Mock.assert_not_called = __mock_assert_not_called # type: ignore def get_top_level_dir() -> Path: @@ -551,4 +563,32 @@ def cloud_init_project_dir(sub_path: str) -> str: return str(get_top_level_dir() / sub_path) +@contextmanager +def does_not_raise(): + """Context manager to parametrize tests raising and not raising exceptions + + Note: In python-3.7+, this can be substituted by contextlib.nullcontext + More info: + https://docs.pytest.org/en/6.2.x/example/parametrize.html?highlight=does_not_raise#parametrizing-conditional-raising + + Example: + -------- + >>> @pytest.mark.parametrize( + >>> "example_input,expectation", + >>> [ + >>> (1, does_not_raise()), + >>> (0, pytest.raises(ZeroDivisionError)), + >>> ], + >>> ) + >>> def test_division(example_input, expectation): + >>> with expectation: + >>> assert (0 / example_input) is not None + + """ + try: + yield + except Exception as ex: + raise pytest.fail("DID RAISE {0}".format(ex)) + + # vi: ts=4 expandtab diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.2653.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.2653.nmconnection new file mode 100644 index 000000000..80483d4f0 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.2653.nmconnection @@ -0,0 +1,21 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init encc000.2653 +uuid=116aaf19-aabc-50ea-b480-e9aee18bda59 +type=vlan +interface-name=encc000.2653 + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[vlan] +id=2653 +parent=f869ebd3-f175-5747-bf02-d0d44d687248 + +[ipv4] +method=manual +may-fail=false +address1=10.245.236.14/24 +gateway=10.245.236.1 +dns=10.245.236.1; diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.nmconnection new file mode 100644 index 000000000..3368388d4 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-encc000.nmconnection @@ -0,0 +1,12 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init encc000 +uuid=f869ebd3-f175-5747-bf02-d0d44d687248 +type=ethernet +interface-name=encc000 + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-en.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-en.nmconnection new file mode 100644 index 000000000..16120bc17 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-en.nmconnection @@ -0,0 +1,16 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init zz-all-en +uuid=159daec9-cba3-5101-85e7-46d831857f43 +type=ethernet +interface-name=zz-all-en + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] + +[ipv4] +method=auto +may-fail=false diff --git a/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-eth.nmconnection b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-eth.nmconnection new file mode 100644 index 000000000..df44d546c --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac/etc/NetworkManager/system-connections/cloud-init-zz-all-eth.nmconnection @@ -0,0 +1,16 @@ +# Generated by cloud-init. Changes will be lost. + +[connection] +id=cloud-init zz-all-eth +uuid=23a83d8a-d7db-5133-a77b-e68a6ac61ec9 +type=ethernet +interface-name=zz-all-eth + +[user] +org.freedesktop.NetworkManager.origin=cloud-init + +[ethernet] + +[ipv4] +method=auto +may-fail=false diff --git a/tests/unittests/net/artifacts/no_matching_mac_v2.yaml b/tests/unittests/net/artifacts/no_matching_mac_v2.yaml new file mode 100644 index 000000000..f5fc5ef10 --- /dev/null +++ b/tests/unittests/net/artifacts/no_matching_mac_v2.yaml @@ -0,0 +1,22 @@ +network: + version: 2 + ethernets: + encc000: {} + zz-all-en: + match: + name: "en*" + dhcp4: true + zz-all-eth: + match: + name: "eth*" + dhcp4: true + vlans: + encc000.2653: + id: 2653 + link: "encc000" + addresses: + - "10.245.236.14/24" + gateway4: "10.245.236.1" + nameservers: + addresses: + - "10.245.236.1" diff --git a/tests/unittests/net/test_dhcp.py b/tests/unittests/net/test_dhcp.py index 08ca001a5..db9f0e972 100644 --- a/tests/unittests/net/test_dhcp.py +++ b/tests/unittests/net/test_dhcp.py @@ -7,7 +7,6 @@ import httpretty import pytest -import cloudinit.net as net from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, NoDHCPLeaseError, @@ -19,6 +18,7 @@ parse_dhcp_lease_file, parse_static_routes, ) +from cloudinit.net.ephemeral import EphemeralDHCPv4 from cloudinit.util import ensure_file, write_file from tests.unittests.helpers import ( CiTestCase, @@ -157,8 +157,8 @@ def test_parse_lease_finds_classless_static_routes(self): write_file(lease_file, content) self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) - @mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network") - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_obtain_lease_parses_static_routes(self, m_maybe, m_ipv4): """EphemeralDHPCv4 parses rfc3442 routes for EphemeralIPv4Network""" lease = [ @@ -173,7 +173,7 @@ def test_obtain_lease_parses_static_routes(self, m_maybe, m_ipv4): } ] m_maybe.return_value = lease - eph = net.dhcp.EphemeralDHCPv4() + eph = EphemeralDHCPv4() eph.obtain_lease() expected_kwargs = { "interface": "wlp3s0", @@ -185,8 +185,8 @@ def test_obtain_lease_parses_static_routes(self, m_maybe, m_ipv4): } m_ipv4.assert_called_with(**expected_kwargs) - @mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network") - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_obtain_centos_lease_parses_static_routes(self, m_maybe, m_ipv4): """ EphemeralDHPCv4 parses rfc3442 routes for EphemeralIPv4Network @@ -204,7 +204,7 @@ def test_obtain_centos_lease_parses_static_routes(self, m_maybe, m_ipv4): } ] m_maybe.return_value = lease - eph = net.dhcp.EphemeralDHCPv4() + eph = EphemeralDHCPv4() eph.obtain_lease() expected_kwargs = { "interface": "wlp3s0", @@ -776,7 +776,7 @@ def test_ephemeral_dhcp_no_network_if_url_connectivity(self, m_dhcp): url = "http://example.org/index.html" httpretty.register_uri(httpretty.GET, url) - with net.dhcp.EphemeralDHCPv4( + with EphemeralDHCPv4( connectivity_url_data={"url": url}, ) as lease: self.assertIsNone(lease) @@ -784,7 +784,7 @@ def test_ephemeral_dhcp_no_network_if_url_connectivity(self, m_dhcp): m_dhcp.assert_not_called() @mock.patch("cloudinit.net.dhcp.subp.subp") - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_ephemeral_dhcp_setup_network_if_url_connectivity( self, m_dhcp, m_subp ): @@ -799,7 +799,7 @@ def test_ephemeral_dhcp_setup_network_if_url_connectivity( m_subp.return_value = ("", "") httpretty.register_uri(httpretty.GET, url, body={}, status=404) - with net.dhcp.EphemeralDHCPv4( + with EphemeralDHCPv4( connectivity_url_data={"url": url}, ) as lease: self.assertEqual(fake_lease, lease) @@ -816,38 +816,38 @@ def test_ephemeral_dhcp_setup_network_if_url_connectivity( ], ) class TestEphemeralDhcpLeaseErrors: - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_obtain_lease_raises_error(self, m_dhcp, error_class): m_dhcp.side_effect = [error_class()] with pytest.raises(error_class): - net.dhcp.EphemeralDHCPv4().obtain_lease() + EphemeralDHCPv4().obtain_lease() assert len(m_dhcp.mock_calls) == 1 - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_obtain_lease_umbrella_error(self, m_dhcp, error_class): m_dhcp.side_effect = [error_class()] with pytest.raises(NoDHCPLeaseError): - net.dhcp.EphemeralDHCPv4().obtain_lease() + EphemeralDHCPv4().obtain_lease() assert len(m_dhcp.mock_calls) == 1 - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_ctx_mgr_raises_error(self, m_dhcp, error_class): m_dhcp.side_effect = [error_class()] with pytest.raises(error_class): - with net.dhcp.EphemeralDHCPv4(): + with EphemeralDHCPv4(): pass assert len(m_dhcp.mock_calls) == 1 - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") def test_ctx_mgr_umbrella_error(self, m_dhcp, error_class): m_dhcp.side_effect = [error_class()] with pytest.raises(NoDHCPLeaseError): - with net.dhcp.EphemeralDHCPv4(): + with EphemeralDHCPv4(): pass assert len(m_dhcp.mock_calls) == 1 diff --git a/tests/unittests/net/test_dns.py b/tests/unittests/net/test_dns.py new file mode 100644 index 000000000..606efecb0 --- /dev/null +++ b/tests/unittests/net/test_dns.py @@ -0,0 +1,32 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from unittest import mock + +from cloudinit import safeyaml +from cloudinit.net import network_state + + +class TestNetDns: + @mock.patch("cloudinit.net.network_state.get_interfaces_by_mac") + @mock.patch("cloudinit.net.get_interfaces_by_mac") + def test_system_mac_address_does_not_break_dns_parsing( + self, by_mac_state, by_mac_init + ): + by_mac_state.return_value = {"00:11:22:33:44:55": "foobar"} + by_mac_init.return_value = {"00:11:22:33:44:55": "foobar"} + state = network_state.parse_net_config_data( + safeyaml.load( + """\ +version: 2 +ethernets: + eth: + match: + macaddress: '00:11:22:33:44:55' + addresses: [10.0.0.2/24] + gateway4: 10.0.0.1 + nameservers: + addresses: [10.0.0.3] +""" + ) + ) + assert "10.0.0.3" in state.dns_nameservers diff --git a/tests/unittests/net/test_ephemeral.py b/tests/unittests/net/test_ephemeral.py new file mode 100644 index 000000000..d2237faf1 --- /dev/null +++ b/tests/unittests/net/test_ephemeral.py @@ -0,0 +1,49 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from unittest import mock + +import pytest + +from cloudinit.net.ephemeral import EphemeralIPNetwork + +M_PATH = "cloudinit.net.ephemeral." + + +class TestEphemeralIPNetwork: + @pytest.mark.parametrize("ipv6", [False, True]) + @pytest.mark.parametrize("ipv4", [False, True]) + @mock.patch(M_PATH + "contextlib.ExitStack") + @mock.patch(M_PATH + "EphemeralIPv6Network") + @mock.patch(M_PATH + "EphemeralDHCPv4") + def test_stack_order( + self, + m_ephemeral_dhcp_v4, + m_ephemeral_ip_v6_network, + m_exit_stack, + ipv4, + ipv6, + ): + interface = object() + with EphemeralIPNetwork(interface, ipv4=ipv4, ipv6=ipv6): + pass + expected_call_args_list = [] + if ipv4: + expected_call_args_list.append( + mock.call(m_ephemeral_dhcp_v4.return_value) + ) + assert [mock.call(interface)] == m_ephemeral_dhcp_v4.call_args_list + else: + assert [] == m_ephemeral_dhcp_v4.call_args_list + if ipv6: + expected_call_args_list.append( + mock.call(m_ephemeral_ip_v6_network.return_value) + ) + assert [ + mock.call(interface) + ] == m_ephemeral_ip_v6_network.call_args_list + else: + assert [] == m_ephemeral_ip_v6_network.call_args_list + assert ( + expected_call_args_list + == m_exit_stack.return_value.enter_context.call_args_list + ) diff --git a/tests/unittests/net/test_init.py b/tests/unittests/net/test_init.py index 35851d281..6feba1e3f 100644 --- a/tests/unittests/net/test_init.py +++ b/tests/unittests/net/test_init.py @@ -13,6 +13,7 @@ import requests import cloudinit.net as net +from cloudinit.net.ephemeral import EphemeralIPv4Network, EphemeralIPv6Network from cloudinit.subp import ProcessExecutionError from cloudinit.util import ensure_file, write_file from tests.unittests.helpers import CiTestCase, HttprettyTestCase @@ -767,7 +768,7 @@ def test_ephemeral_ipv4_network_errors_on_missing_params(self, m_subp): params = copy.deepcopy(required_params) params[key] = None with self.assertRaises(ValueError) as context_manager: - net.EphemeralIPv4Network(**params) + EphemeralIPv4Network(**params) error = context_manager.exception self.assertIn("Cannot init network on", str(error)) self.assertEqual(0, m_subp.call_count) @@ -783,7 +784,7 @@ def test_ephemeral_ipv4_network_errors_invalid_mask_prefix(self, m_subp): for error_val in invalid_masks: params["prefix_or_mask"] = error_val with self.assertRaises(ValueError) as context_manager: - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): pass error = context_manager.exception self.assertIn( @@ -849,7 +850,7 @@ def test_ephemeral_ipv4_network_performs_teardown(self, m_subp): "prefix_or_mask": "255.255.255.0", "broadcast": "192.168.2.255", } - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): self.assertEqual(expected_setup_calls, m_subp.call_args_list) m_subp.assert_has_calls(expected_teardown_calls) @@ -867,7 +868,7 @@ def test_ephemeral_ipv4_no_network_if_url_connectivity( "connectivity_url_data": {"url": "http://example.org/index.html"}, } - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): self.assertEqual( [mock.call(url="http://example.org/index.html", timeout=5)], m_readurl.call_args_list, @@ -907,7 +908,7 @@ def test_ephemeral_ipv4_network_noop_when_configured(self, m_subp): update_env={"LANG": "C"}, ) ] - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): pass self.assertEqual(expected_calls, m_subp.call_args_list) self.assertIn( @@ -925,7 +926,7 @@ def test_ephemeral_ipv4_network_with_prefix(self, m_subp): } for prefix_val in ["24", 16]: # prefix can be int or string params["prefix_or_mask"] = prefix_val - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): pass m_subp.assert_has_calls( [ @@ -1050,7 +1051,7 @@ def test_ephemeral_ipv4_network_with_new_default_route(self, m_subp): ), ] - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): self.assertEqual(expected_setup_calls, m_subp.call_args_list) m_subp.assert_has_calls(expected_teardown_calls) @@ -1189,11 +1190,26 @@ def test_ephemeral_ipv4_network_with_rfc3442_static_routes(self, m_subp): capture=True, ), ] - with net.EphemeralIPv4Network(**params): + with EphemeralIPv4Network(**params): self.assertEqual(expected_setup_calls, m_subp.call_args_list) m_subp.assert_has_calls(expected_setup_calls + expected_teardown_calls) +class TestEphemeralIPV6Network: + @mock.patch("cloudinit.net.read_sys_net") + @mock.patch("cloudinit.net.subp.subp") + def test_ephemeral_ipv6_network_performs_setup(self, m_subp, _): + """EphemeralIPv4Network performs teardown on the device if setup.""" + expected_setup_calls = [ + mock.call( + ["ip", "link", "set", "dev", "eth0", "up"], + capture=False, + ), + ] + with EphemeralIPv6Network(interface="eth0"): + assert expected_setup_calls == m_subp.call_args_list + + class TestHasURLConnectivity(HttprettyTestCase): def setUp(self): super(TestHasURLConnectivity, self).setUp() diff --git a/tests/unittests/net/test_net_rendering.py b/tests/unittests/net/test_net_rendering.py new file mode 100644 index 000000000..06feab891 --- /dev/null +++ b/tests/unittests/net/test_net_rendering.py @@ -0,0 +1,101 @@ +"""Home of the tests for end-to-end net rendering + +Tests defined here should take a v1 or v2 yaml config as input, and verify +that the rendered network config is as expected. Input files are defined +under `tests/unittests/net/artifacts` with the format of + +.yaml + +For example, if my test name is "test_all_the_things" and I'm testing a +v2 format, I should have a file named test_all_the_things_v2.yaml. + +If a renderer outputs multiple files, the expected files should live in +the artifacts directory under the given test name. For example, if I'm +expecting NetworkManager to output a file named eth0.nmconnection as +part of my "test_all_the_things" test, then in the artifacts directory +there should be a +`test_all_the_things/etc/NetworkManager/system-connections/eth0.nmconnection` +file. + +To add a new nominal test, create the input and output files, then add the test +name to the `test_convert` test along with it's supported renderers. + +Before adding a test here, check that it is not already represented +in `unittests/test_net.py`. While that file contains similar tests, it has +become too large to be maintainable. +""" +import glob +from enum import Flag, auto +from pathlib import Path + +import pytest + +from cloudinit import safeyaml +from cloudinit.net.netplan import Renderer as NetplanRenderer +from cloudinit.net.network_manager import Renderer as NetworkManagerRenderer +from cloudinit.net.network_state import NetworkState, parse_net_config_data + +ARTIFACT_DIR = Path(__file__).parent.absolute() / "artifacts" + + +class Renderer(Flag): + Netplan = auto() + NetworkManager = auto() + Networkd = auto() + + +@pytest.fixture(autouse=True) +def setup(mocker): + mocker.patch("cloudinit.net.network_state.get_interfaces_by_mac") + + +def _check_netplan( + network_state: NetworkState, netplan_path: Path, expected_config +): + if network_state.version == 2: + renderer = NetplanRenderer(config={"netplan_path": netplan_path}) + renderer.render_network_state(network_state) + assert safeyaml.load(netplan_path.read_text()) == expected_config, ( + f"Netplan config generated at {netplan_path} does not match v2 " + "config defined for this test." + ) + else: + raise NotImplementedError + + +def _check_network_manager(network_state: NetworkState, tmp_path: Path): + renderer = NetworkManagerRenderer() + renderer.render_network_state( + network_state, target=str(tmp_path / "no_matching_mac") + ) + expected_paths = glob.glob( + str(ARTIFACT_DIR / "no_matching_mac" / "**/*.nmconnection"), + recursive=True, + ) + for expected_path in expected_paths: + expected_contents = Path(expected_path).read_text() + actual_path = tmp_path / expected_path.split( + str(ARTIFACT_DIR), maxsplit=1 + )[1].lstrip("/") + assert ( + actual_path.exists() + ), f"Expected {actual_path} to exist, but it does not" + actual_contents = actual_path.read_text() + assert expected_contents.strip() == actual_contents.strip() + + +@pytest.mark.parametrize( + "test_name, renderers", + [("no_matching_mac_v2", Renderer.Netplan | Renderer.NetworkManager)], +) +def test_convert(test_name, renderers, tmp_path): + network_config = safeyaml.load( + Path(ARTIFACT_DIR, f"{test_name}.yaml").read_text() + ) + network_state = parse_net_config_data(network_config["network"]) + if Renderer.Netplan in renderers: + _check_netplan( + network_state, tmp_path / "netplan.yaml", network_config + ) + if Renderer.NetworkManager in renderers: + _check_network_manager(network_state, tmp_path) diff --git a/tests/unittests/net/test_network_state.py b/tests/unittests/net/test_network_state.py index ec21d007f..75d033dc8 100644 --- a/tests/unittests/net/test_network_state.py +++ b/tests/unittests/net/test_network_state.py @@ -79,7 +79,8 @@ def test_version_2_passes_self_as_config(self): ncfg = {"version": 2, "otherconfig": {}, "somemore": [1, 2, 3]} network_state.parse_net_config_data(ncfg) self.assertEqual( - [mock.call(version=2, config=ncfg)], self.m_nsi.call_args_list + [mock.call(version=2, config=ncfg, renderer=None)], + self.m_nsi.call_args_list, ) def test_valid_config_gets_network_state(self): @@ -101,17 +102,19 @@ def test_empty_v2_config_gets_network_state(self): class TestNetworkStateParseConfigV2(CiTestCase): def test_version_2_ignores_renderer_key(self): ncfg = {"version": 2, "renderer": "networkd", "ethernets": {}} - nsi = network_state.NetworkStateInterpreter( - version=ncfg["version"], config=ncfg - ) - nsi.parse_config(skip_broken=False) - self.assertEqual(ncfg, nsi.as_dict()["config"]) + with mock.patch("cloudinit.net.network_state.get_interfaces_by_mac"): + nsi = network_state.NetworkStateInterpreter( + version=ncfg["version"], config=ncfg + ) + nsi.parse_config(skip_broken=False) + self.assertEqual(ncfg, nsi.as_dict()["config"]) class TestNetworkStateParseNameservers: def _parse_network_state_from_config(self, config): - yaml = safeyaml.load(config) - return network_state.parse_net_config_data(yaml["network"]) + with mock.patch("cloudinit.net.network_state.get_interfaces_by_mac"): + yaml = safeyaml.load(config) + return network_state.parse_net_config_data(yaml["network"]) def test_v1_nameservers_valid(self): config = self._parse_network_state_from_config( @@ -136,7 +139,9 @@ def test_v1_nameservers_invalid(self): V1_CONFIG_NAMESERVERS_INVALID ) - def test_v2_nameservers(self): + def test_v2_nameservers(self, mocker): + mocker.patch("cloudinit.net.network_state.get_interfaces_by_mac") + mocker.patch("cloudinit.net.get_interfaces_by_mac") config = self._parse_network_state_from_config(V2_CONFIG_NAMESERVERS) # Ensure DNS defined on interface exists on interface diff --git a/tests/unittests/net/test_networkd.py b/tests/unittests/net/test_networkd.py index ec1d04e9b..a22c50929 100644 --- a/tests/unittests/net/test_networkd.py +++ b/tests/unittests/net/test_networkd.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + from cloudinit import safeyaml from cloudinit.net import network_state, networkd @@ -10,6 +12,7 @@ eth0: match: macaddress: '00:11:22:33:44:55' + addresses: [172.16.10.2/12, 172.16.10.3/12] nameservers: search: [spam.local, eggs.local] addresses: [8.8.8.8] @@ -22,7 +25,13 @@ addresses: [4.4.4.4] """ -V2_CONFIG_SET_NAME_RENDERED_ETH0 = """[Match] +V2_CONFIG_SET_NAME_RENDERED_ETH0 = """[Address] +Address=172.16.10.2/12 + +[Address] +Address=172.16.10.3/12 + +[Match] MACAddress=00:11:22:33:44:55 Name=eth0 @@ -47,13 +56,15 @@ class TestNetworkdRenderState: def _parse_network_state_from_config(self, config): - yaml = safeyaml.load(config) - return network_state.parse_net_config_data(yaml["network"]) + with mock.patch("cloudinit.net.network_state.get_interfaces_by_mac"): + yaml = safeyaml.load(config) + return network_state.parse_net_config_data(yaml["network"]) def test_networkd_render_with_set_name(self): - ns = self._parse_network_state_from_config(V2_CONFIG_SET_NAME) - renderer = networkd.Renderer() - rendered_content = renderer._render_content(ns) + with mock.patch("cloudinit.net.get_interfaces_by_mac"): + ns = self._parse_network_state_from_config(V2_CONFIG_SET_NAME) + renderer = networkd.Renderer() + rendered_content = renderer._render_content(ns) assert "eth0" in rendered_content assert rendered_content["eth0"] == V2_CONFIG_SET_NAME_RENDERED_ETH0 diff --git a/tests/unittests/test_reporting.py b/tests/unittests/reporting/test_reporting.py similarity index 79% rename from tests/unittests/test_reporting.py rename to tests/unittests/reporting/test_reporting.py index f6dd96e0a..a6cf6a955 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/reporting/test_reporting.py @@ -4,9 +4,16 @@ from unittest import mock +import pytest + from cloudinit import reporting +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) from cloudinit.reporting import events, handlers -from tests.unittests.helpers import TestCase +from tests.unittests.helpers import TestCase, skipUnlessJsonSchema def _fake_registry(): @@ -453,4 +460,110 @@ def test_invalid_status_access_raises_value_error(self): self.assertRaises(AttributeError, getattr, events.status, "BOGUS") -# vi: ts=4 expandtab +@skipUnlessJsonSchema() +class TestReportingSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + # GOOD: Minimum valid parameters + ({"reporting": {"a": {"type": "print"}}}, None), + ({"reporting": {"a": {"type": "log"}}}, None), + ( + { + "reporting": { + "a": {"type": "webhook", "endpoint": "http://a"} + } + }, + None, + ), + ({"reporting": {"a": {"type": "hyperv"}}}, None), + # GOOD: All valid parameters + ({"reporting": {"a": {"type": "log", "level": "WARN"}}}, None), + ( + { + "reporting": { + "a": { + "type": "webhook", + "endpoint": "http://a", + "timeout": 1, + "retries": 1, + "consumer_key": "somekey", + "token_key": "somekey", + "token_secret": "somesecret", + "consumer_secret": "somesecret", + } + } + }, + None, + ), + ( + { + "reporting": { + "a": { + "type": "hyperv", + "kvp_file_path": "/some/path", + "event_types": ["a", "b"], + } + } + }, + None, + ), + # GOOD: All combined together + ( + { + "reporting": { + "a": {"type": "print"}, + "b": {"type": "log", "level": "WARN"}, + "c": { + "type": "webhook", + "endpoint": "http://a", + "timeout": 1, + "retries": 1, + "consumer_key": "somekey", + "token_key": "somekey", + "token_secret": "somesecret", + "consumer_secret": "somesecret", + }, + "d": { + "type": "hyperv", + "kvp_file_path": "/some/path", + "event_types": ["a", "b"], + }, + } + }, + None, + ), + # BAD: no top level objects + ({"reporting": "a"}, "'a' is not of type 'object'"), + ({"reporting": {"a": "b"}}, "'b' is not of type 'object'"), + # BAD: invalid type + ({"reporting": {"a": {"type": "b"}}}, "not valid"), + # BAD: invalid additional properties + ({"reporting": {"a": {"type": "print", "a": "b"}}}, "not valid"), + ({"reporting": {"a": {"type": "log", "a": "b"}}}, "not valid"), + ( + { + "reporting": { + "a": { + "type": "webhook", + "endpoint": "http://a", + "a": "b", + } + } + }, + "not valid", + ), + ({"reporting": {"a": {"type": "hyperv", "a": "b"}}}, "not valid"), + # BAD: missing required properties + ({"reporting": {"a": {"level": "FATAL"}}}, "not valid"), + ({"reporting": {"a": {"endpoint": "http://a"}}}, "not valid"), + ({"reporting": {"a": {"kvp_file_path": "/a/b"}}}, "not valid"), + ({"reporting": {"a": {"type": "webhook"}}}, "not valid"), + ], + ) + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/reporting/test_reporting_hyperv.py similarity index 100% rename from tests/unittests/test_reporting_hyperv.py rename to tests/unittests/reporting/test_reporting_hyperv.py diff --git a/tests/unittests/reporting/test_webhook_handler.py b/tests/unittests/reporting/test_webhook_handler.py new file mode 100644 index 000000000..2df71d935 --- /dev/null +++ b/tests/unittests/reporting/test_webhook_handler.py @@ -0,0 +1,121 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import time +from contextlib import suppress +from unittest.mock import PropertyMock + +import pytest +import responses + +from cloudinit.reporting import flush_events +from cloudinit.reporting.events import report_start_event +from cloudinit.reporting.handlers import WebHookHandler + + +class TestWebHookHandler: + @pytest.fixture(autouse=True) + def setup(self, mocker): + handler = WebHookHandler(endpoint="http://localhost") + m_registered_items = mocker.patch( + "cloudinit.registry.DictRegistry.registered_items", + new_callable=PropertyMock, + ) + m_registered_items.return_value = {"webhook": handler} + + @responses.activate + def test_webhook_handler(self, caplog): + """Test the happy path.""" + responses.add(responses.POST, "http://localhost", status=200) + report_start_event("name", "description") + flush_events() + assert 1 == caplog.text.count( + "Read from http://localhost (200, 0b) after 1 attempts" + ) + + @responses.activate + def test_404(self, caplog): + """Test failure""" + responses.add(responses.POST, "http://localhost", status=404) + report_start_event("name", "description") + flush_events() + assert 1 == caplog.text.count("Failed posting event") + + @responses.activate + def test_background_processing(self, caplog): + """Test that processing happens in background. + + In the non-flush case, ensure that the event is still posted. + Since the event is posted in the background, wait while looping. + """ + responses.add(responses.POST, "http://localhost", status=200) + report_start_event("name", "description") + start_time = time.time() + while time.time() - start_time < 3: + with suppress(AssertionError): + assert ( + "Read from http://localhost (200, 0b) after 1 attempts" + in caplog.text + ) + break + else: + pytest.fail("Never got expected log message") + + @responses.activate + @pytest.mark.parametrize( + "num_failures,expected_log_count,expected_cancel", + [(2, 2, False), (3, 3, True), (50, 3, True)], + ) + def test_failures_cancel_flush( + self, caplog, num_failures, expected_log_count, expected_cancel + ): + """Test that too many failures will cancel further processing on flush. + + 2 messages should not cancel on flush + 3 or more should cancel on flush + The number of received messages will be based on how many have + been processed before the flush was initiated. + """ + responses.add(responses.POST, "http://localhost", status=404) + for _ in range(num_failures): + report_start_event("name", "description") + flush_events() + # Force a context switch. Without this, it's possible that the + # expected log message hasn't made it to the log file yet + time.sleep(0.01) + + # If we've pushed a bunch of messages, any number could have been + # processed before we get to the flush. + assert ( + expected_log_count + <= caplog.text.count("Failed posting event") + <= num_failures + ) + cancelled_message = ( + "Multiple consecutive failures in WebHookHandler. " + "Cancelling all queued events" + ) + if expected_cancel: + assert cancelled_message in caplog.text + else: + assert cancelled_message not in caplog.text + + @responses.activate + def test_multiple_failures_no_flush(self, caplog): + """Test we don't cancel posting if flush hasn't been requested. + + Since processing happens in the background, wait in a loop + for all messages to be posted + """ + responses.add(responses.POST, "http://localhost", status=404) + for _ in range(10): + report_start_event("name", "description") + start_time = time.time() + while time.time() - start_time < 3: + with suppress(AssertionError): + assert 10 == caplog.text.count("Failed posting event") + break + time.sleep(0.01) # Force context switch + else: + pytest.fail( + "Expected 20 failures, only got " + f"{caplog.text.count('Failed posting event')}" + ) diff --git a/tests/unittests/runs/test_simple_run.py b/tests/unittests/runs/test_simple_run.py index 2b51117cf..7b364a3e8 100644 --- a/tests/unittests/runs/test_simple_run.py +++ b/tests/unittests/runs/test_simple_run.py @@ -23,7 +23,10 @@ def setUp(self): "datasource_list": ["None"], "runcmd": ["ls /etc"], # test ALL_DISTROS "spacewalk": {}, # test non-ubuntu distros module definition - "system_info": {"paths": {"run_dir": self.new_root}}, + "system_info": { + "paths": {"run_dir": self.new_root}, + "distro": "ubuntu", + }, "write_files": [ { "path": "/etc/blah.ini", diff --git a/tests/unittests/sources/test_aliyun.py b/tests/unittests/sources/test_aliyun.py index 8a61d5eec..e628dc027 100644 --- a/tests/unittests/sources/test_aliyun.py +++ b/tests/unittests/sources/test_aliyun.py @@ -149,7 +149,7 @@ def _test_get_iid(self): def _test_host_name(self): self.assertEqual( - self.default_metadata["hostname"], self.ds.get_hostname() + self.default_metadata["hostname"], self.ds.get_hostname().hostname ) @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index a83da6c92..a88a6b1f0 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -12,7 +12,6 @@ import httpretty import pytest import requests -import yaml from cloudinit import distros, helpers, subp, url_helper from cloudinit.net import dhcp @@ -23,7 +22,6 @@ from cloudinit.util import ( MountFailedError, b64e, - decode_binary, json_dumps, load_file, load_json, @@ -37,17 +35,15 @@ mock, populate_dir, resourceLocation, - wrap_and_call, ) MOCKPATH = "cloudinit.sources.DataSourceAzure." @pytest.fixture -def azure_ds(patched_data_dir_path, paths): +def azure_ds(patched_data_dir_path, mock_dmi_read_dmi_data, paths): """Provide DataSourceAzure instance with mocks for minimal test case.""" - with mock.patch(MOCKPATH + "_is_platform_viable", return_value=True): - yield dsaz.DataSourceAzure(sys_cfg={}, distro=mock.Mock(), paths=paths) + yield dsaz.DataSourceAzure(sys_cfg={}, distro=mock.Mock(), paths=paths) @pytest.fixture @@ -86,6 +82,35 @@ def mock_azure_report_failure_to_fabric(): yield m +@pytest.fixture +def mock_chassis_asset_tag(): + with mock.patch.object( + dsaz.ChassisAssetTag, + "query_system", + return_value=dsaz.ChassisAssetTag.AZURE_CLOUD.value, + ) as m: + yield m + + +@pytest.fixture +def mock_device_driver(): + with mock.patch( + MOCKPATH + "device_driver", + autospec=True, + return_value=None, + ) as m: + yield m + + +@pytest.fixture +def mock_generate_fallback_config(): + with mock.patch( + MOCKPATH + "net.generate_fallback_config", + autospec=True, + ) as m: + yield m + + @pytest.fixture def mock_time(): with mock.patch( @@ -100,6 +125,8 @@ def mock_dmi_read_dmi_data(): def fake_read(key: str) -> str: if key == "system-uuid": return "fake-system-uuid" + elif key == "chassis-asset-tag": + return "7783-7084-3265-9085-8269-3286-77" raise RuntimeError() with mock.patch( @@ -122,7 +149,7 @@ def mock_ephemeral_dhcp_v4(): @pytest.fixture def mock_net_dhcp_maybe_perform_dhcp_discovery(): with mock.patch( - "cloudinit.net.dhcp.maybe_perform_dhcp_discovery", + "cloudinit.net.ephemeral.maybe_perform_dhcp_discovery", return_value=[ { "unknown-245": "0a:0b:0c:0d", @@ -140,7 +167,7 @@ def mock_net_dhcp_maybe_perform_dhcp_discovery(): @pytest.fixture def mock_net_dhcp_EphemeralIPv4Network(): with mock.patch( - "cloudinit.net.dhcp.EphemeralIPv4Network", + "cloudinit.net.ephemeral.EphemeralIPv4Network", autospec=True, ) as m: yield m @@ -279,83 +306,101 @@ def patched_markers_dir_path(tmpdir): @pytest.fixture -def patched_reported_ready_marker_path(patched_markers_dir_path): +def patched_reported_ready_marker_path(azure_ds, patched_markers_dir_path): reported_ready_marker = patched_markers_dir_path / "reported_ready" - with mock.patch( - MOCKPATH + "REPORTED_READY_MARKER_FILE", str(reported_ready_marker) + with mock.patch.object( + azure_ds, "_reported_ready_marker_file", str(reported_ready_marker) ): yield reported_ready_marker -def construct_valid_ovf_env( - data=None, pubkeys=None, userdata=None, platform_settings=None +def construct_ovf_env( + *, + custom_data=None, + hostname="test-host", + username="test-user", + password=None, + public_keys=None, + disable_ssh_password_auth=None, + preprovisioned_vm=None, + preprovisioned_vm_type=None, ): - if data is None: - data = {"HostName": "FOOHOST"} - if pubkeys is None: - pubkeys = {} - - content = """ - - - 1.0 - - LinuxProvisioningConfiguration - """ - for key, dval in data.items(): - if isinstance(dval, dict): - val = dict(dval).get("text") - attrs = " " + " ".join( - [ - "%s='%s'" % (k, v) - for k, v in dict(dval).items() - if k != "text" - ] - ) - else: - val = dval - attrs = "" - content += "<%s%s>%s\n" % (key, attrs, val, key) - - if userdata: - content += "%s\n" % (b64e(userdata)) - - if pubkeys: - content += "\n" - for fp, path, value in pubkeys: - content += " " - if fp and path: - content += "%s%s" % ( - fp, - path, - ) - if value: - content += "%s" % value - content += "\n" - content += "" - content += """ - - - 1.0 - - kms.core.windows.net - false - """ - if platform_settings: - for k, v in platform_settings.items(): - content += "<%s>%s\n" % (k, v, k) - if "PreprovisionedVMType" not in platform_settings: - content += """""" - content += """ -""" - - return content + content = [ + '', + '', + "", + "1.0", + "", + "" + "LinuxProvisioningConfiguration" + "", + ] + if hostname is not None: + content.append("%s" % hostname) + if username is not None: + content.append("%s" % username) + if password is not None: + content.append("%s" % password) + if custom_data is not None: + content.append( + "%s" % (b64e(custom_data)) + ) + if disable_ssh_password_auth is not None: + content.append( + "%s" + % str(disable_ssh_password_auth).lower() + + "" + ) + if public_keys is not None: + content += ["", ""] + for public_key in public_keys: + content.append("") + fp = public_key.get("fingerprint") + if fp is not None: + content.append("%s" % fp) + path = public_key.get("path") + if path is not None: + content.append("%s" % path) + value = public_key.get("value") + if value is not None: + content.append("%s" % value) + content.append("") + content += ["", ""] + content += [ + "", + "", + "", + "1.0", + "", + "" + "kms.core.windows.net" + "", + "false", + '', + ] + if preprovisioned_vm is not None: + content.append( + "%s" + % str(preprovisioned_vm).lower() + ) + + if preprovisioned_vm_type is None: + content.append('') + else: + content.append( + "%s" + % preprovisioned_vm_type + ) + content += [ + "", + "", + "", + ] + + return "\n".join(content) NETWORK_METADATA = { @@ -441,9 +486,200 @@ def construct_valid_ovf_env( EXAMPLE_UUID = "d0df4c54-4ecb-4a4b-9954-5bdf3ed5c3b8" -class TestParseNetworkConfig(CiTestCase): +class TestGenerateNetworkConfig: + @pytest.mark.parametrize( + "label,metadata,expected", + [ + ( + "simple interface", + NETWORK_METADATA["network"], + { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": {"name": "eth0"}, + "set-name": "eth0", + } + }, + "version": 2, + }, + ), + ( + "multiple interfaces with increasing route metric", + { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": {"ipAddress": []}, + "ipv4": { + "subnet": [ + {"prefix": "24", "address": "10.0.0.0"} + ], + "ipAddress": [ + { + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "104.46.124.81", + } + ], + }, + } + ] + * 3 + }, + { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": {"name": "eth0"}, + "set-name": "eth0", + }, + "eth1": { + "set-name": "eth1", + "match": {"name": "eth1"}, + "dhcp6": False, + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 200}, + }, + "eth2": { + "set-name": "eth2", + "match": {"name": "eth2"}, + "dhcp6": False, + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 300}, + }, + }, + "version": 2, + }, + ), + ( + "secondary IPv4s are static", + { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": { + "subnet": [ + { + "prefix": "10", + "address": "2001:dead:beef::16", + } + ], + "ipAddress": [ + {"privateIpAddress": "2001:dead:beef::1"} + ], + }, + "ipv4": { + "subnet": [ + {"prefix": "24", "address": "10.0.0.0"}, + ], + "ipAddress": [ + { + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "104.46.124.81", + }, + { + "privateIpAddress": "11.0.0.5", + "publicIpAddress": "104.46.124.82", + }, + { + "privateIpAddress": "12.0.0.6", + "publicIpAddress": "104.46.124.83", + }, + ], + }, + } + ] + }, + { + "ethernets": { + "eth0": { + "addresses": ["11.0.0.5/24", "12.0.0.6/24"], + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": True, + "dhcp6-overrides": {"route-metric": 100}, + "match": {"name": "eth0"}, + "set-name": "eth0", + } + }, + "version": 2, + }, + ), + ( + "ipv6 secondaries", + { + "interface": [ + { + "macAddress": "000D3A047598", + "ipv6": { + "subnet": [ + { + "prefix": "10", + "address": "2001:dead:beef::16", + } + ], + "ipAddress": [ + {"privateIpAddress": "2001:dead:beef::1"}, + {"privateIpAddress": "2001:dead:beef::2"}, + ], + }, + } + ] + }, + { + "ethernets": { + "eth0": { + "addresses": ["2001:dead:beef::2/10"], + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": True, + "dhcp6-overrides": {"route-metric": 100}, + "match": {"name": "eth0"}, + "set-name": "eth0", + } + }, + "version": 2, + }, + ), + ], + ) + def test_parsing_scenarios( + self, label, mock_device_driver, metadata, expected + ): + assert ( + dsaz.generate_network_config_from_instance_network_metadata( + metadata + ) + == expected + ) + + def test_match_hv_netvsc(self, mock_device_driver): + mock_device_driver.return_value = "hv_netvsc" - maxDiff = None + assert dsaz.generate_network_config_from_instance_network_metadata( + NETWORK_METADATA["network"] + ) == { + "ethernets": { + "eth0": { + "dhcp4": True, + "dhcp4-overrides": {"route-metric": 100}, + "dhcp6": False, + "match": { + "name": "eth0", + "driver": "hv_netvsc", + }, + "set-name": "eth0", + } + }, + "version": 2, + } + + +class TestNetworkConfig: fallback_config = { "version": 1, "config": [ @@ -457,11 +693,8 @@ class TestParseNetworkConfig(CiTestCase): ], } - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - def test_single_ipv4_nic_configuration(self, m_driver): - """parse_network_config emits dhcp on single nic with ipv4""" + def test_single_ipv4_nic_configuration(self, azure_ds, mock_device_driver): + """Network config emits dhcp on single nic with ipv4""" expected = { "ethernets": { "eth0": { @@ -474,198 +707,31 @@ def test_single_ipv4_nic_configuration(self, m_driver): }, "version": 2, } - self.assertEqual(expected, dsaz.parse_network_config(NETWORK_METADATA)) - - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - def test_increases_route_metric_for_non_primary_nics(self, m_driver): - """parse_network_config increases route-metric for each nic""" - expected = { - "ethernets": { - "eth0": { - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": False, - "match": {"name": "eth0"}, - "set-name": "eth0", - }, - "eth1": { - "set-name": "eth1", - "match": {"name": "eth1"}, - "dhcp6": False, - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 200}, - }, - "eth2": { - "set-name": "eth2", - "match": {"name": "eth2"}, - "dhcp6": False, - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 300}, - }, - }, - "version": 2, - } - imds_data = copy.deepcopy(NETWORK_METADATA) - imds_data["network"]["interface"].append(SECONDARY_INTERFACE) - third_intf = copy.deepcopy(SECONDARY_INTERFACE) - third_intf["macAddress"] = third_intf["macAddress"].replace("22", "33") - third_intf["ipv4"]["subnet"][0]["address"] = "10.0.2.0" - third_intf["ipv4"]["ipAddress"][0]["privateIpAddress"] = "10.0.2.6" - imds_data["network"]["interface"].append(third_intf) - self.assertEqual(expected, dsaz.parse_network_config(imds_data)) + azure_ds._metadata_imds = NETWORK_METADATA - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - def test_ipv4_and_ipv6_route_metrics_match_for_nics(self, m_driver): - """parse_network_config emits matching ipv4 and ipv6 route-metrics.""" - expected = { - "ethernets": { - "eth0": { - "addresses": ["10.0.0.5/24", "2001:dead:beef::2/128"], - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": True, - "dhcp6-overrides": {"route-metric": 100}, - "match": {"name": "eth0"}, - "set-name": "eth0", - }, - "eth1": { - "set-name": "eth1", - "match": {"name": "eth1"}, - "dhcp4": True, - "dhcp6": False, - "dhcp4-overrides": {"route-metric": 200}, - }, - "eth2": { - "set-name": "eth2", - "match": {"name": "eth2"}, - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 300}, - "dhcp6": True, - "dhcp6-overrides": {"route-metric": 300}, - }, - }, - "version": 2, - } - imds_data = copy.deepcopy(NETWORK_METADATA) - nic1 = imds_data["network"]["interface"][0] - nic1["ipv4"]["ipAddress"].append({"privateIpAddress": "10.0.0.5"}) - - nic1["ipv6"] = { - "subnet": [{"address": "2001:dead:beef::16"}], - "ipAddress": [ - {"privateIpAddress": "2001:dead:beef::1"}, - {"privateIpAddress": "2001:dead:beef::2"}, - ], - } - imds_data["network"]["interface"].append(SECONDARY_INTERFACE) - third_intf = copy.deepcopy(SECONDARY_INTERFACE) - third_intf["macAddress"] = third_intf["macAddress"].replace("22", "33") - third_intf["ipv4"]["subnet"][0]["address"] = "10.0.2.0" - third_intf["ipv4"]["ipAddress"][0]["privateIpAddress"] = "10.0.2.6" - third_intf["ipv6"] = { - "subnet": [{"prefix": "64", "address": "2001:dead:beef::2"}], - "ipAddress": [{"privateIpAddress": "2001:dead:beef::1"}], - } - imds_data["network"]["interface"].append(third_intf) - self.assertEqual(expected, dsaz.parse_network_config(imds_data)) + assert azure_ds.network_config == expected - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - def test_ipv4_secondary_ips_will_be_static_addrs(self, m_driver): - """parse_network_config emits primary ipv4 as dhcp others are static""" - expected = { - "ethernets": { - "eth0": { - "addresses": ["10.0.0.5/24"], - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": True, - "dhcp6-overrides": {"route-metric": 100}, - "match": {"name": "eth0"}, - "set-name": "eth0", - } - }, - "version": 2, - } - imds_data = copy.deepcopy(NETWORK_METADATA) - nic1 = imds_data["network"]["interface"][0] - nic1["ipv4"]["ipAddress"].append({"privateIpAddress": "10.0.0.5"}) + def test_uses_fallback_cfg_when_apply_network_config_is_false( + self, azure_ds, mock_device_driver, mock_generate_fallback_config + ): + azure_ds.ds_cfg["apply_network_config"] = False + azure_ds._metadata_imds = NETWORK_METADATA + mock_generate_fallback_config.return_value = self.fallback_config - nic1["ipv6"] = { - "subnet": [{"prefix": "10", "address": "2001:dead:beef::16"}], - "ipAddress": [{"privateIpAddress": "2001:dead:beef::1"}], - } - self.assertEqual(expected, dsaz.parse_network_config(imds_data)) + assert azure_ds.network_config == self.fallback_config - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - def test_ipv6_secondary_ips_will_be_static_cidrs(self, m_driver): - """parse_network_config emits primary ipv6 as dhcp others are static""" - expected = { - "ethernets": { - "eth0": { - "addresses": ["10.0.0.5/24", "2001:dead:beef::2/10"], - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": True, - "dhcp6-overrides": {"route-metric": 100}, - "match": {"name": "eth0"}, - "set-name": "eth0", - } - }, - "version": 2, - } - imds_data = copy.deepcopy(NETWORK_METADATA) - nic1 = imds_data["network"]["interface"][0] - nic1["ipv4"]["ipAddress"].append({"privateIpAddress": "10.0.0.5"}) - - # Secondary ipv6 addresses currently ignored/unconfigured - nic1["ipv6"] = { - "subnet": [{"prefix": "10", "address": "2001:dead:beef::16"}], - "ipAddress": [ - {"privateIpAddress": "2001:dead:beef::1"}, - {"privateIpAddress": "2001:dead:beef::2"}, - ], - } - self.assertEqual(expected, dsaz.parse_network_config(imds_data)) + def test_uses_fallback_cfg_when_imds_metadata_unset( + self, azure_ds, mock_device_driver, mock_generate_fallback_config + ): + azure_ds._metadata_imds = UNSET + mock_generate_fallback_config.return_value = self.fallback_config - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", - return_value="hv_netvsc", - ) - def test_match_driver_for_netvsc(self, m_driver): - """parse_network_config emits driver when using netvsc.""" - expected = { - "ethernets": { - "eth0": { - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": False, - "match": { - "name": "eth0", - "driver": "hv_netvsc", - }, - "set-name": "eth0", - } - }, - "version": 2, - } - self.assertEqual(expected, dsaz.parse_network_config(NETWORK_METADATA)) + assert azure_ds.network_config == self.fallback_config - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - @mock.patch("cloudinit.net.generate_fallback_config") - def test_parse_network_config_uses_fallback_cfg_when_no_network_metadata( - self, m_fallback_config, m_driver + def test_uses_fallback_cfg_when_no_network_metadata( + self, azure_ds, mock_device_driver, mock_generate_fallback_config ): - """parse_network_config generates fallback network config when the + """Network config generates fallback network config when the IMDS instance metadata is corrupted/invalid, such as when network metadata is not present. """ @@ -673,20 +739,15 @@ def test_parse_network_config_uses_fallback_cfg_when_no_network_metadata( NETWORK_METADATA ) del imds_metadata_missing_network_metadata["network"] - m_fallback_config.return_value = self.fallback_config - self.assertEqual( - self.fallback_config, - dsaz.parse_network_config(imds_metadata_missing_network_metadata), - ) + mock_generate_fallback_config.return_value = self.fallback_config + azure_ds._metadata_imds = imds_metadata_missing_network_metadata - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - @mock.patch("cloudinit.net.generate_fallback_config") - def test_parse_network_config_uses_fallback_cfg_when_no_interface_metadata( - self, m_fallback_config, m_driver + assert azure_ds.network_config == self.fallback_config + + def test_uses_fallback_cfg_when_no_interface_metadata( + self, azure_ds, mock_device_driver, mock_generate_fallback_config ): - """parse_network_config generates fallback network config when the + """Network config generates fallback network config when the IMDS instance metadata is corrupted/invalid, such as when network interface metadata is not present. """ @@ -694,13 +755,10 @@ def test_parse_network_config_uses_fallback_cfg_when_no_interface_metadata( NETWORK_METADATA ) del imds_metadata_missing_interface_metadata["network"]["interface"] - m_fallback_config.return_value = self.fallback_config - self.assertEqual( - self.fallback_config, - dsaz.parse_network_config( - imds_metadata_missing_interface_metadata - ), - ) + mock_generate_fallback_config.return_value = self.fallback_config + azure_ds._metadata_imds = imds_metadata_missing_interface_metadata + + assert azure_ds.network_config == self.fallback_config class TestGetMetadataFromIMDS(HttprettyTestCase): @@ -1002,7 +1060,6 @@ def _load_possible_azure_ds(seed_dir, cache_dir): dsaz.BUILTIN_DS_CONFIG["data_dir"] = self.waagent_d - self.m_is_platform_viable = mock.MagicMock(autospec=True) self.m_get_metadata_from_fabric = mock.MagicMock(return_value=[]) self.m_report_failure_to_fabric = mock.MagicMock(autospec=True) self.m_get_interfaces = mock.MagicMock( @@ -1026,6 +1083,10 @@ def _dmi_mocks(key): return self.instance_id elif key == "chassis-asset-tag": return "7783-7084-3265-9085-8269-3286-77" + raise RuntimeError() + + self.m_read_dmi_data = mock.MagicMock(autospec=True) + self.m_read_dmi_data.side_effect = _dmi_mocks self.apply_patches( [ @@ -1034,7 +1095,6 @@ def _dmi_mocks(key): "list_possible_azure_ds", self.m_list_possible_azure_ds, ), - (dsaz, "_is_platform_viable", self.m_is_platform_viable), ( dsaz, "get_metadata_from_fabric", @@ -1061,7 +1121,7 @@ def _dmi_mocks(key): ( dsaz.dmi, "read_dmi_data", - mock.MagicMock(side_effect=_dmi_mocks), + self.m_read_dmi_data, ), ( dsaz.util, @@ -1131,14 +1191,16 @@ def test_not_is_platform_viable_seed_should_return_no_datasource(self): # Return a non-matching asset tag value data = {} dsrc = self._get_ds(data) - self.m_is_platform_viable.return_value = False + self.m_read_dmi_data.side_effect = lambda x: "notazure" with mock.patch.object( dsrc, "crawl_metadata" ) as m_crawl_metadata, mock.patch.object( dsrc, "_report_failure" ) as m_report_failure: ret = dsrc.get_data() - self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) + assert self.m_read_dmi_data.mock_calls == [ + mock.call("chassis-asset-tag") + ] self.assertFalse(ret) # Assert that for non viable platforms, # there is no communication with the Azure datasource. @@ -1155,27 +1217,22 @@ def test_platform_viable_but_no_devs_should_return_no_datasource(self): data = {} dsrc = self._get_ds(data) with mock.patch.object(dsrc, "_report_failure") as m_report_failure: - self.m_is_platform_viable.return_value = True ret = dsrc.get_data() - self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) self.assertFalse(ret) self.assertEqual(1, m_report_failure.call_count) def test_crawl_metadata_exception_returns_no_datasource(self): data = {} dsrc = self._get_ds(data) - self.m_is_platform_viable.return_value = True with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: m_crawl_metadata.side_effect = Exception ret = dsrc.get_data() - self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) self.assertEqual(1, m_crawl_metadata.call_count) self.assertFalse(ret) def test_crawl_metadata_exception_should_report_failure_with_msg(self): data = {} dsrc = self._get_ds(data) - self.m_is_platform_viable.return_value = True with mock.patch.object( dsrc, "crawl_metadata" ) as m_crawl_metadata, mock.patch.object( @@ -1191,7 +1248,6 @@ def test_crawl_metadata_exception_should_report_failure_with_msg(self): def test_crawl_metadata_exc_should_log_could_not_crawl_msg(self): data = {} dsrc = self._get_ds(data) - self.m_is_platform_viable.return_value = True with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: m_crawl_metadata.side_effect = Exception dsrc.get_data() @@ -1201,16 +1257,15 @@ def test_crawl_metadata_exc_should_log_could_not_crawl_msg(self): ) def test_basic_seed_dir(self): - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(hostname="myhost"), "sys_cfg": {}, } dsrc = self._get_ds(data) ret = dsrc.get_data() self.assertTrue(ret) self.assertEqual(dsrc.userdata_raw, "") - self.assertEqual(dsrc.metadata["local-hostname"], odata["HostName"]) + self.assertEqual(dsrc.metadata["local-hostname"], "myhost") self.assertTrue( os.path.isfile(os.path.join(self.waagent_d, "ovf-env.xml")) ) @@ -1221,9 +1276,8 @@ def test_basic_seed_dir(self): ) def test_data_dir_without_imds_data(self): - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(hostname="myhost"), "sys_cfg": {}, } dsrc = self._get_ds( @@ -1240,7 +1294,7 @@ def test_data_dir_without_imds_data(self): self.assertTrue(ret) self.assertEqual(dsrc.userdata_raw, "") - self.assertEqual(dsrc.metadata["local-hostname"], odata["HostName"]) + self.assertEqual(dsrc.metadata["local-hostname"], "myhost") self.assertTrue( os.path.isfile(os.path.join(self.waagent_d, "ovf-env.xml")) ) @@ -1269,9 +1323,10 @@ def test_basic_dev_file(self): def test_get_data_non_ubuntu_will_not_remove_network_scripts(self): """get_data on non-Ubuntu will not remove ubuntu net scripts.""" - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env( + hostname="myhost", username="myuser" + ), "sys_cfg": {}, } @@ -1282,9 +1337,8 @@ def test_get_data_non_ubuntu_will_not_remove_network_scripts(self): def test_get_data_on_ubuntu_will_remove_network_scripts(self): """get_data will remove ubuntu net scripts on Ubuntu distro.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } @@ -1295,9 +1349,8 @@ def test_get_data_on_ubuntu_will_remove_network_scripts(self): def test_get_data_on_ubuntu_will_not_remove_network_scripts_disabled(self): """When apply_network_config false, do not remove scripts on Ubuntu.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": False}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } @@ -1307,28 +1360,19 @@ def test_get_data_on_ubuntu_will_not_remove_network_scripts_disabled(self): def test_crawl_metadata_returns_structured_data_and_caches_nothing(self): """Return all structured metadata and cache no class attributes.""" - yaml_cfg = "" - odata = { - "HostName": "myhost", - "UserName": "myuser", - "UserData": {"text": "FOOBAR", "encoding": "plain"}, - "dscfg": {"text": yaml_cfg, "encoding": "plain"}, - } data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env( + hostname="myhost", username="myuser", custom_data="FOOBAR" + ), "sys_cfg": {}, } dsrc = self._get_ds(data) expected_cfg = { "PreprovisionedVMType": None, "PreprovisionedVm": False, - "datasource": {"Azure": {}}, "system_info": {"default_user": {"name": "myuser"}}, } expected_metadata = { - "azure_data": { - "configurationsettype": "LinuxProvisioningConfiguration" - }, "imds": NETWORK_METADATA, "instance-id": EXAMPLE_UUID, "local-hostname": "myhost", @@ -1346,11 +1390,11 @@ def test_crawl_metadata_returns_structured_data_and_caches_nothing(self): list(crawled_metadata["files"].keys()), ["ovf-env.xml"] ) self.assertIn( - b"myhost", + b"myhost", crawled_metadata["files"]["ovf-env.xml"], ) self.assertEqual(crawled_metadata["metadata"], expected_metadata) - self.assertEqual(crawled_metadata["userdata_raw"], "FOOBAR") + self.assertEqual(crawled_metadata["userdata_raw"], b"FOOBAR") self.assertEqual(dsrc.userdata_raw, None) self.assertEqual(dsrc.metadata, {}) self.assertEqual(dsrc._metadata_imds, UNSET) @@ -1372,9 +1416,7 @@ def test_crawl_metadata_raises_invalid_metadata_on_error(self): def test_crawl_metadata_call_imds_once_no_reprovision(self): """If reprovisioning, report ready at the end""" - ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "False"} - ) + ovfenv = construct_ovf_env(preprovisioned_vm=False) data = {"ovfcontent": ovfenv, "sys_cfg": {}} dsrc = self._get_ds(data) @@ -1390,9 +1432,7 @@ def test_crawl_metadata_call_imds_twice_with_reprovision( self, poll_imds_func, m_report_ready, m_write ): """If reprovisioning, imds metadata will be fetched twice""" - ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"} - ) + ovfenv = construct_ovf_env(preprovisioned_vm=True) data = {"ovfcontent": ovfenv, "sys_cfg": {}} dsrc = self._get_ds(data) @@ -1409,9 +1449,7 @@ def test_crawl_metadata_on_reprovision_reports_ready( self, poll_imds_func, m_report_ready, m_write ): """If reprovisioning, report ready at the end""" - ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"} - ) + ovfenv = construct_ovf_env(preprovisioned_vm=True) data = {"ovfcontent": ovfenv, "sys_cfg": {}} dsrc = self._get_ds(data) @@ -1432,11 +1470,8 @@ def test_crawl_metadata_waits_for_nic_on_savable_vms( self, detect_nics, poll_imds_func, report_ready_func, m_write ): """If reprovisioning, report ready at the end""" - ovfenv = construct_valid_ovf_env( - platform_settings={ - "PreprovisionedVMType": "Savable", - "PreprovisionedVm": "True", - } + ovfenv = construct_ovf_env( + preprovisioned_vm=True, preprovisioned_vm_type="Savable" ) data = {"ovfcontent": ovfenv, "sys_cfg": {}} @@ -1459,9 +1494,7 @@ def test_crawl_metadata_on_reprovision_reports_ready_using_lease( self, m_readurl, m_report_ready, m_media_switch, m_write ): """If reprovisioning, report ready using the obtained lease""" - ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"} - ) + ovfenv = construct_ovf_env(preprovisioned_vm=True) data = {"ovfcontent": ovfenv, "sys_cfg": {}} dsrc = self._get_ds(data) @@ -1476,7 +1509,7 @@ def test_crawl_metadata_on_reprovision_reports_ready_using_lease( self.m_dhcp.return_value.obtain_lease.return_value = lease m_media_switch.return_value = None - reprovision_ovfenv = construct_valid_ovf_env() + reprovision_ovfenv = construct_ovf_env() m_readurl.return_value = url_helper.StringResponse( reprovision_ovfenv.encode("utf-8") ) @@ -1490,7 +1523,7 @@ def test_crawl_metadata_on_reprovision_reports_ready_using_lease( def test_waagent_d_has_0700_perms(self): # we expect /var/lib/waagent to be created 0700 - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) ret = dsrc.get_data() self.assertTrue(ret) self.assertTrue(os.path.isdir(self.waagent_d)) @@ -1502,9 +1535,8 @@ def test_waagent_d_has_0700_perms(self): def test_network_config_set_from_imds(self, m_driver): """Datasource.network_config returns IMDS network data.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } expected_network_config = { @@ -1531,9 +1563,8 @@ def test_network_config_set_from_imds_route_metric_for_secondary_nic( ): """Datasource.network_config adds route-metric to secondary nics.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } expected_network_config = { @@ -1583,9 +1614,8 @@ def test_network_config_set_from_imds_for_secondary_nic_no_ip( ): """If an IP address is empty then there should no config for it.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } expected_network_config = { @@ -1610,9 +1640,8 @@ def test_network_config_set_from_imds_for_secondary_nic_no_ip( def test_availability_zone_set_from_imds(self): """Datasource.availability returns IMDS platformFaultDomain.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -1622,9 +1651,8 @@ def test_availability_zone_set_from_imds(self): def test_region_set_from_imds(self): """Datasource.region returns IMDS region location.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -1638,7 +1666,7 @@ def test_sys_cfg_set_never_destroy_ntfs(self): } } data = { - "ovfcontent": construct_valid_ovf_env(data={}), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } @@ -1651,8 +1679,7 @@ def test_sys_cfg_set_never_destroy_ntfs(self): ) def test_username_used(self): - odata = {"HostName": "myhost", "UserName": "myuser"} - data = {"ovfcontent": construct_valid_ovf_env(data=odata)} + data = {"ovfcontent": construct_ovf_env(username="myuser")} dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -1661,13 +1688,14 @@ def test_username_used(self): dsrc.cfg["system_info"]["default_user"]["name"], "myuser" ) + assert "ssh_pwauth" not in dsrc.cfg + def test_password_given(self): - odata = { - "HostName": "myhost", - "UserName": "myuser", - "UserPassword": "mypass", + data = { + "ovfcontent": construct_ovf_env( + username="myuser", password="mypass" + ) } - data = {"ovfcontent": construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -1676,27 +1704,107 @@ def test_password_given(self): defuser = dsrc.cfg["system_info"]["default_user"] # default user should be updated username and should not be locked. - self.assertEqual(defuser["name"], odata["UserName"]) + self.assertEqual(defuser["name"], "myuser") self.assertFalse(defuser["lock_passwd"]) # passwd is crypt formated string $id$salt$encrypted # encrypting plaintext with salt value of everything up to final '$' # should equal that after the '$' - pos = defuser["passwd"].rfind("$") + 1 + pos = defuser["hashed_passwd"].rfind("$") + 1 self.assertEqual( - defuser["passwd"], - crypt.crypt(odata["UserPassword"], defuser["passwd"][0:pos]), + defuser["hashed_passwd"], + crypt.crypt("mypass", defuser["hashed_passwd"][0:pos]), ) - # the same hashed value should also be present in cfg['password'] - self.assertEqual(defuser["passwd"], dsrc.cfg["password"]) + assert dsrc.cfg["ssh_pwauth"] is True + + def test_password_with_disable_ssh_pw_auth_true(self): + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + password="mypass", + disable_ssh_password_auth=True, + ) + } + + dsrc = self._get_ds(data) + dsrc.get_data() + + assert dsrc.cfg["ssh_pwauth"] is False + + def test_password_with_disable_ssh_pw_auth_false(self): + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + password="mypass", + disable_ssh_password_auth=False, + ) + } + + dsrc = self._get_ds(data) + dsrc.get_data() + + assert dsrc.cfg["ssh_pwauth"] is True + + def test_password_with_disable_ssh_pw_auth_unspecified(self): + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + password="mypass", + disable_ssh_password_auth=None, + ) + } + + dsrc = self._get_ds(data) + dsrc.get_data() + + assert dsrc.cfg["ssh_pwauth"] is True + + def test_no_password_with_disable_ssh_pw_auth_true(self): + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + disable_ssh_password_auth=True, + ) + } + + dsrc = self._get_ds(data) + dsrc.get_data() + + assert dsrc.cfg["ssh_pwauth"] is False + + def test_no_password_with_disable_ssh_pw_auth_false(self): + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + disable_ssh_password_auth=False, + ) + } + + dsrc = self._get_ds(data) + dsrc.get_data() + + assert dsrc.cfg["ssh_pwauth"] is True + + def test_no_password_with_disable_ssh_pw_auth_unspecified(self): + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + disable_ssh_password_auth=None, + ) + } + + dsrc = self._get_ds(data) + dsrc.get_data() + + assert "ssh_pwauth" not in dsrc.cfg def test_user_not_locked_if_password_redacted(self): - odata = { - "HostName": "myhost", - "UserName": "myuser", - "UserPassword": dsaz.DEF_PASSWD_REDACTION, + data = { + "ovfcontent": construct_ovf_env( + username="myuser", + password=dsaz.DEF_PASSWD_REDACTION, + ) } - data = {"ovfcontent": construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -1705,24 +1813,13 @@ def test_user_not_locked_if_password_redacted(self): defuser = dsrc.cfg["system_info"]["default_user"] # default user should be updated username and should not be locked. - self.assertEqual(defuser["name"], odata["UserName"]) + self.assertEqual(defuser["name"], "myuser") self.assertIn("lock_passwd", defuser) self.assertFalse(defuser["lock_passwd"]) - def test_userdata_plain(self): - mydata = "FOOBAR" - odata = {"UserData": {"text": mydata, "encoding": "plain"}} - data = {"ovfcontent": construct_valid_ovf_env(data=odata)} - - dsrc = self._get_ds(data) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(decode_binary(dsrc.userdata_raw), mydata) - def test_userdata_found(self): mydata = "FOOBAR" - odata = {"UserData": {"text": b64e(mydata), "encoding": "base64"}} - data = {"ovfcontent": construct_valid_ovf_env(data=odata)} + data = {"ovfcontent": construct_ovf_env(custom_data=mydata)} dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -1731,9 +1828,8 @@ def test_userdata_found(self): def test_default_ephemeral_configs_ephemeral_exists(self): # make sure the ephemeral configs are correct if disk present - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": {}, } @@ -1761,9 +1857,8 @@ def changed_exists(path): def test_default_ephemeral_configs_ephemeral_does_not_exist(self): # make sure the ephemeral configs are correct if disk not present - odata = {} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": {}, } @@ -1783,34 +1878,9 @@ def changed_exists(path): assert "disk_setup" not in cfg assert "fs_setup" not in cfg - def test_provide_disk_aliases(self): - # Make sure that user can affect disk aliases - dscfg = {"disk_aliases": {"ephemeral0": "/dev/sdc"}} - odata = { - "HostName": "myhost", - "UserName": "myuser", - "dscfg": {"text": b64e(yaml.dump(dscfg)), "encoding": "base64"}, - } - usercfg = { - "disk_setup": { - "/dev/sdc": {"something": "..."}, - "ephemeral0": False, - } - } - userdata = "#cloud-config" + yaml.dump(usercfg) + "\n" - - ovfcontent = construct_valid_ovf_env(data=odata, userdata=userdata) - data = {"ovfcontent": ovfcontent, "sys_cfg": {}} - - dsrc = self._get_ds(data) - ret = dsrc.get_data() - self.assertTrue(ret) - cfg = dsrc.get_config_obj() - self.assertTrue(cfg) - def test_userdata_arrives(self): userdata = "This is my user-data" - xml = construct_valid_ovf_env(data={}, userdata=userdata) + xml = construct_ovf_env(custom_data=userdata) data = {"ovfcontent": xml} dsrc = self._get_ds(data) dsrc.get_data() @@ -1818,12 +1888,11 @@ def test_userdata_arrives(self): self.assertEqual(userdata.encode("us-ascii"), dsrc.userdata_raw) def test_password_redacted_in_ovf(self): - odata = { - "HostName": "myhost", - "UserName": "myuser", - "UserPassword": "mypass", + data = { + "ovfcontent": construct_ovf_env( + username="myuser", password="mypass" + ) } - data = {"ovfcontent": construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -1846,7 +1915,7 @@ def test_password_redacted_in_ovf(self): self.assertEqual(dsaz.DEF_PASSWD_REDACTION, elem.text) def test_ovf_env_arrives_in_waagent_dir(self): - xml = construct_valid_ovf_env(data={}, userdata="FOODATA") + xml = construct_ovf_env(custom_data="FOODATA") dsrc = self._get_ds({"ovfcontent": xml}) dsrc.get_data() @@ -1857,18 +1926,18 @@ def test_ovf_env_arrives_in_waagent_dir(self): self.xml_equals(xml, load_file(ovf_env_path)) def test_ovf_can_include_unicode(self): - xml = construct_valid_ovf_env(data={}) + xml = construct_ovf_env() xml = "\ufeff{0}".format(xml) dsrc = self._get_ds({"ovfcontent": xml}) dsrc.get_data() def test_dsaz_report_ready_returns_true_when_report_succeeds(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) assert dsrc._report_ready() == [] @mock.patch(MOCKPATH + "report_diagnostic_event") def test_dsaz_report_ready_failure_reports_telemetry(self, m_report_diag): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) self.m_get_metadata_from_fabric.side_effect = Exception("foo") with pytest.raises(Exception): @@ -1883,7 +1952,7 @@ def test_dsaz_report_ready_failure_reports_telemetry(self, m_report_diag): ] def test_dsaz_report_failure_returns_true_when_report_succeeds(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: # mock crawl metadata failure to cause report failure @@ -1895,7 +1964,7 @@ def test_dsaz_report_failure_returns_true_when_report_succeeds(self): def test_dsaz_report_failure_returns_false_and_does_not_propagate_exc( self, ): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object( dsrc, "crawl_metadata" @@ -1923,7 +1992,7 @@ def test_dsaz_report_failure_returns_false_and_does_not_propagate_exc( self.assertEqual(2, self.m_report_failure_to_fabric.call_count) def test_dsaz_report_failure_description_msg(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: # mock crawl metadata failure to cause report failure @@ -1936,7 +2005,7 @@ def test_dsaz_report_failure_description_msg(self): ) def test_dsaz_report_failure_no_description_msg(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: m_crawl_metadata.side_effect = Exception @@ -1947,7 +2016,7 @@ def test_dsaz_report_failure_no_description_msg(self): ) def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object( dsrc, "crawl_metadata" @@ -1965,7 +2034,7 @@ def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): ) def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: # mock crawl metadata failure to cause report failure @@ -1988,13 +2057,13 @@ def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): def test_exception_fetching_fabric_data_doesnt_propagate(self): """Errors communicating with fabric should warn, but return True.""" - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) self.m_get_metadata_from_fabric.side_effect = Exception ret = self._get_and_setup(dsrc) self.assertTrue(ret) def test_fabric_data_included_in_metadata(self): - dsrc = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) self.m_get_metadata_from_fabric.return_value = ["ssh-key-value"] ret = self._get_and_setup(dsrc) self.assertTrue(ret) @@ -2006,7 +2075,7 @@ def test_instance_id_case_insensitive(self): upper_iid = EXAMPLE_UUID.upper() # lowercase current UUID ds = self._get_ds( - {"ovfcontent": construct_valid_ovf_env()}, instance_id=lower_iid + {"ovfcontent": construct_ovf_env()}, instance_id=lower_iid ) # UPPERCASE previous write_file( @@ -2018,7 +2087,7 @@ def test_instance_id_case_insensitive(self): # UPPERCASE current UUID ds = self._get_ds( - {"ovfcontent": construct_valid_ovf_env()}, instance_id=upper_iid + {"ovfcontent": construct_ovf_env()}, instance_id=upper_iid ) # lowercase previous write_file( @@ -2030,7 +2099,7 @@ def test_instance_id_case_insensitive(self): def test_instance_id_endianness(self): """Return the previous iid when dmi uuid is the byteswapped iid.""" - ds = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) + ds = self._get_ds({"ovfcontent": construct_ovf_env()}) # byte-swapped previous write_file( os.path.join(self.paths.cloud_dir, "data", "instance-id"), @@ -2042,164 +2111,50 @@ def test_instance_id_endianness(self): ) # not byte-swapped previous write_file( - os.path.join(self.paths.cloud_dir, "data", "instance-id"), - "644CDFD0-CB4E-4B4A-9954-5BDF3ED5C3B8", - ) - ds.get_data() - self.assertEqual(self.instance_id, ds.metadata["instance-id"]) - - def test_instance_id_from_dmidecode_used(self): - ds = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) - ds.get_data() - self.assertEqual(self.instance_id, ds.metadata["instance-id"]) - - def test_instance_id_from_dmidecode_used_for_builtin(self): - ds = self._get_ds({"ovfcontent": construct_valid_ovf_env()}) - ds.get_data() - self.assertEqual(self.instance_id, ds.metadata["instance-id"]) - - @mock.patch(MOCKPATH + "util.is_FreeBSD") - @mock.patch(MOCKPATH + "_check_freebsd_cdrom") - def test_list_possible_azure_ds(self, m_check_fbsd_cdrom, m_is_FreeBSD): - """On FreeBSD, possible devs should show /dev/cd0.""" - m_is_FreeBSD.return_value = True - m_check_fbsd_cdrom.return_value = True - possible_ds = [] - for src in dsaz.list_possible_azure_ds("seed_dir", "cache_dir"): - possible_ds.append(src) - self.assertEqual( - possible_ds, - [ - "seed_dir", - dsaz.DEFAULT_PROVISIONING_ISO_DEV, - "/dev/cd0", - "cache_dir", - ], - ) - self.assertEqual( - [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list - ) - - @mock.patch( - "cloudinit.sources.DataSourceAzure.device_driver", return_value=None - ) - @mock.patch("cloudinit.net.generate_fallback_config") - def test_imds_network_config(self, mock_fallback, m_driver): - """Network config is generated from IMDS network data when present.""" - sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} - data = { - "ovfcontent": construct_valid_ovf_env(data=odata), - "sys_cfg": sys_cfg, - } - - dsrc = self._get_ds(data) - ret = dsrc.get_data() - self.assertTrue(ret) - - expected_cfg = { - "ethernets": { - "eth0": { - "dhcp4": True, - "dhcp4-overrides": {"route-metric": 100}, - "dhcp6": False, - "match": {"name": "eth0"}, - "set-name": "eth0", - } - }, - "version": 2, - } - - self.assertEqual(expected_cfg, dsrc.network_config) - mock_fallback.assert_not_called() - - @mock.patch("cloudinit.net.get_interface_mac") - @mock.patch("cloudinit.net.get_devicelist") - @mock.patch("cloudinit.net.device_driver") - @mock.patch("cloudinit.net.generate_fallback_config") - def test_imds_network_ignored_when_apply_network_config_false( - self, mock_fallback, mock_dd, mock_devlist, mock_get_mac - ): - """When apply_network_config is False, use fallback instead of IMDS.""" - sys_cfg = {"datasource": {"Azure": {"apply_network_config": False}}} - odata = {"HostName": "myhost", "UserName": "myuser"} - data = { - "ovfcontent": construct_valid_ovf_env(data=odata), - "sys_cfg": sys_cfg, - } - fallback_config = { - "version": 1, - "config": [ - { - "type": "physical", - "name": "eth0", - "mac_address": "00:11:22:33:44:55", - "params": {"driver": "hv_netsvc"}, - "subnets": [{"type": "dhcp"}], - } - ], - } - mock_fallback.return_value = fallback_config - - mock_devlist.return_value = ["eth0"] - mock_dd.return_value = ["hv_netsvc"] - mock_get_mac.return_value = "00:11:22:33:44:55" - - dsrc = self._get_ds(data) - self.assertTrue(dsrc.get_data()) - self.assertEqual(dsrc.network_config, fallback_config) - - @mock.patch("cloudinit.net.get_interface_mac") - @mock.patch("cloudinit.net.get_devicelist") - @mock.patch("cloudinit.net.device_driver") - @mock.patch("cloudinit.net.generate_fallback_config", autospec=True) - def test_fallback_network_config( - self, mock_fallback, mock_dd, mock_devlist, mock_get_mac - ): - """On absent IMDS network data, generate network fallback config.""" - odata = {"HostName": "myhost", "UserName": "myuser"} - data = { - "ovfcontent": construct_valid_ovf_env(data=odata), - "sys_cfg": {}, - } - - fallback_config = { - "version": 1, - "config": [ - { - "type": "physical", - "name": "eth0", - "mac_address": "00:11:22:33:44:55", - "params": {"driver": "hv_netsvc"}, - "subnets": [{"type": "dhcp"}], - } - ], - } - mock_fallback.return_value = fallback_config + os.path.join(self.paths.cloud_dir, "data", "instance-id"), + "644CDFD0-CB4E-4B4A-9954-5BDF3ED5C3B8", + ) + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata["instance-id"]) - mock_devlist.return_value = ["eth0"] - mock_dd.return_value = ["hv_netsvc"] - mock_get_mac.return_value = "00:11:22:33:44:55" + def test_instance_id_from_dmidecode_used(self): + ds = self._get_ds({"ovfcontent": construct_ovf_env()}) + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata["instance-id"]) - dsrc = self._get_ds(data) - # Represent empty response from network imds - self.m_get_metadata_from_imds.return_value = {} - ret = dsrc.get_data() - self.assertTrue(ret) + def test_instance_id_from_dmidecode_used_for_builtin(self): + ds = self._get_ds({"ovfcontent": construct_ovf_env()}) + ds.get_data() + self.assertEqual(self.instance_id, ds.metadata["instance-id"]) - netconfig = dsrc.network_config - self.assertEqual(netconfig, fallback_config) - mock_fallback.assert_called_with( - blacklist_drivers=["mlx4_core", "mlx5_core"], config_driver=True + @mock.patch(MOCKPATH + "util.is_FreeBSD") + @mock.patch(MOCKPATH + "_check_freebsd_cdrom") + def test_list_possible_azure_ds(self, m_check_fbsd_cdrom, m_is_FreeBSD): + """On FreeBSD, possible devs should show /dev/cd0.""" + m_is_FreeBSD.return_value = True + m_check_fbsd_cdrom.return_value = True + possible_ds = [] + for src in dsaz.list_possible_azure_ds("seed_dir", "cache_dir"): + possible_ds.append(src) + self.assertEqual( + possible_ds, + [ + "seed_dir", + dsaz.DEFAULT_PROVISIONING_ISO_DEV, + "/dev/cd0", + "cache_dir", + ], + ) + self.assertEqual( + [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list ) @mock.patch(MOCKPATH + "net.get_interfaces", autospec=True) def test_blacklist_through_distro(self, m_net_get_interfaces): """Verify Azure DS updates blacklist drivers in the distro's networking object.""" - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": {}, } @@ -2221,9 +2176,8 @@ def test_blacklist_through_distro(self, m_net_get_interfaces): ) def test_get_public_ssh_keys_with_imds(self, m_parse_certificates): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -2256,9 +2210,8 @@ def test_get_public_ssh_keys_with_no_openssh_format( imds_data["compute"]["publicKeys"][0]["keyData"] = "no-openssh-format" m_get_metadata_from_imds.return_value = imds_data sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -2272,9 +2225,8 @@ def test_get_public_ssh_keys_with_no_openssh_format( def test_get_public_ssh_keys_without_imds(self, m_get_metadata_from_imds): m_get_metadata_from_imds.return_value = dict() sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -2295,9 +2247,8 @@ def get_metadata_from_imds_side_eff(*args, **kwargs): m_get_metadata_from_imds.side_effect = get_metadata_from_imds_side_eff sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -2326,9 +2277,8 @@ def get_metadata_from_imds_side_eff(*args, **kwargs): ) def test_imds_api_version_wanted_exists(self, m_get_metadata_from_imds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } dsrc = self._get_ds(data) @@ -2348,9 +2298,8 @@ def test_imds_api_version_wanted_exists(self, m_get_metadata_from_imds): @mock.patch(MOCKPATH + "get_metadata_from_imds") def test_hostname_from_imds(self, m_get_metadata_from_imds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) @@ -2367,9 +2316,8 @@ def test_hostname_from_imds(self, m_get_metadata_from_imds): @mock.patch(MOCKPATH + "get_metadata_from_imds") def test_username_from_imds(self, m_get_metadata_from_imds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) @@ -2388,9 +2336,8 @@ def test_username_from_imds(self, m_get_metadata_from_imds): @mock.patch(MOCKPATH + "get_metadata_from_imds") def test_disable_password_from_imds(self, m_get_metadata_from_imds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) @@ -2407,9 +2354,8 @@ def test_disable_password_from_imds(self, m_get_metadata_from_imds): @mock.patch(MOCKPATH + "get_metadata_from_imds") def test_userdata_from_imds(self, m_get_metadata_from_imds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} - odata = {"HostName": "myhost", "UserName": "myuser"} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } userdata = "userdataImds" @@ -2431,14 +2377,9 @@ def test_userdata_from_imds_with_customdata_from_OVF( self, m_get_metadata_from_imds ): userdataOVF = "userdataOVF" - odata = { - "HostName": "myhost", - "UserName": "myuser", - "UserData": {"text": b64e(userdataOVF), "encoding": "base64"}, - } sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { - "ovfcontent": construct_valid_ovf_env(data=odata), + "ovfcontent": construct_ovf_env(custom_data=userdataOVF), "sys_cfg": sys_cfg, } @@ -2487,18 +2428,17 @@ def test_wb_invalid_ovf_env_xml_calls_read_azure_ovf(self): class TestReadAzureOvf(CiTestCase): def test_invalid_xml_raises_non_azure_ds(self): - invalid_xml = "" + construct_valid_ovf_env(data={}) + invalid_xml = "" + construct_ovf_env() self.assertRaises( dsaz.BrokenAzureDataSource, dsaz.read_azure_ovf, invalid_xml ) def test_load_with_pubkeys(self): - mypklist = [{"fingerprint": "fp1", "path": "path1", "value": ""}] - pubkeys = [(x["fingerprint"], x["path"], x["value"]) for x in mypklist] - content = construct_valid_ovf_env(pubkeys=pubkeys) + public_keys = [{"fingerprint": "fp1", "path": "path1", "value": ""}] + content = construct_ovf_env(public_keys=public_keys) (_md, _ud, cfg) = dsaz.read_azure_ovf(content) - for mypk in mypklist: - self.assertIn(mypk, cfg["_pubkeys"]) + for pk in public_keys: + self.assertIn(pk, cfg["_pubkeys"]) class TestCanDevBeReformatted(CiTestCase): @@ -2866,9 +2806,7 @@ class TestPreprovisioningReadAzureOvfFlag(CiTestCase): def test_read_azure_ovf_with_true_flag(self): """The read_azure_ovf method should set the PreprovisionedVM cfg flag if the proper setting is present.""" - content = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"} - ) + content = construct_ovf_env(preprovisioned_vm=True) ret = dsaz.read_azure_ovf(content) cfg = ret[2] self.assertTrue(cfg["PreprovisionedVm"]) @@ -2876,9 +2814,7 @@ def test_read_azure_ovf_with_true_flag(self): def test_read_azure_ovf_with_false_flag(self): """The read_azure_ovf method should set the PreprovisionedVM cfg flag to false if the proper setting is false.""" - content = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "False"} - ) + content = construct_ovf_env(preprovisioned_vm=False) ret = dsaz.read_azure_ovf(content) cfg = ret[2] self.assertFalse(cfg["PreprovisionedVm"]) @@ -2886,7 +2822,7 @@ def test_read_azure_ovf_with_false_flag(self): def test_read_azure_ovf_without_flag(self): """The read_azure_ovf method should not set the PreprovisionedVM cfg flag.""" - content = construct_valid_ovf_env() + content = construct_ovf_env() ret = dsaz.read_azure_ovf(content) cfg = ret[2] self.assertFalse(cfg["PreprovisionedVm"]) @@ -2895,11 +2831,8 @@ def test_read_azure_ovf_without_flag(self): def test_read_azure_ovf_with_running_type(self): """The read_azure_ovf method should set PreprovisionedVMType cfg flag to Running.""" - content = construct_valid_ovf_env( - platform_settings={ - "PreprovisionedVMType": "Running", - "PreprovisionedVm": "True", - } + content = construct_ovf_env( + preprovisioned_vm=True, preprovisioned_vm_type="Running" ) ret = dsaz.read_azure_ovf(content) cfg = ret[2] @@ -2909,11 +2842,8 @@ def test_read_azure_ovf_with_running_type(self): def test_read_azure_ovf_with_savable_type(self): """The read_azure_ovf method should set PreprovisionedVMType cfg flag to Savable.""" - content = construct_valid_ovf_env( - platform_settings={ - "PreprovisionedVMType": "Savable", - "PreprovisionedVm": "True", - } + content = construct_ovf_env( + preprovisioned_vm=True, preprovisioned_vm_type="Savable" ) ret = dsaz.read_azure_ovf(content) cfg = ret[2] @@ -2997,7 +2927,7 @@ def test_determine_pps_with_reprovision_marker( == dsaz.PPSType.UNKNOWN ) assert is_file.mock_calls == [ - mock.call(dsaz.REPORTED_READY_MARKER_FILE) + mock.call(azure_ds._reported_ready_marker_file) ] @@ -3014,10 +2944,7 @@ def setUp(self): def test_reprovision_calls__poll_imds(self, _poll_imds, isfile): """_reprovision will poll IMDS.""" isfile.return_value = False - hostname = "myhost" - username = "myuser" - odata = {"HostName": hostname, "UserName": username} - _poll_imds.return_value = construct_valid_ovf_env(data=odata) + _poll_imds.return_value = construct_ovf_env() dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) dsa._reprovision() _poll_imds.assert_called_with() @@ -3053,7 +2980,7 @@ def test_detect_nic_attach_reports_ready_and_waits_for_detach( self.assertEqual(1, m_detach.call_count) self.assertEqual(1, m_writefile.call_count) m_writefile.assert_called_with( - dsaz.REPORTED_READY_MARKER_FILE, mock.ANY + dsa._reported_ready_marker_file, mock.ANY ) @mock.patch(MOCKPATH + "util.write_file", autospec=True) @@ -3231,8 +3158,8 @@ def test_wait_for_all_nics_ready_raises_if_socket_fails(self, m_socket): @mock.patch("cloudinit.net.find_fallback_nic", return_value="eth9") -@mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network") -@mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") +@mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") +@mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") @mock.patch( "cloudinit.sources.helpers.netlink.wait_for_media_disconnect_connect" ) @@ -3288,7 +3215,9 @@ def fake_timeout_once(**kwargs): m_request.side_effect = fake_timeout_once dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) - with mock.patch(MOCKPATH + "REPORTED_READY_MARKER_FILE", report_file): + with mock.patch.object( + dsa, "_reported_ready_marker_file", report_file + ): dsa._poll_imds() assert m_report_ready.mock_calls == [mock.call()] @@ -3316,7 +3245,9 @@ def test_poll_imds_skips_dhcp_if_ctx_present( m_isfile.return_value = True dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) dsa._ephemeral_dhcp_ctx = mock.Mock(lease={}) - with mock.patch(MOCKPATH + "REPORTED_READY_MARKER_FILE", report_file): + with mock.patch.object( + dsa, "_reported_ready_marker_file", report_file + ): dsa._poll_imds() self.assertEqual(0, m_dhcp.call_count) self.assertEqual(0, m_media_switch.call_count) @@ -3353,8 +3284,8 @@ def fake_timeout_once(**kwargs): report_file = self.tmp_path("report_marker", self.tmp) m_isfile.return_value = True dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) - with mock.patch( - MOCKPATH + "REPORTED_READY_MARKER_FILE", report_file + with mock.patch.object( + dsa, "_reported_ready_marker_file", report_file ), mock.patch.object(dsa, "_ephemeral_dhcp_ctx") as m_dhcp_ctx: m_dhcp_ctx.obtain_lease.return_value = "Dummy lease" dsa._ephemeral_dhcp_ctx = m_dhcp_ctx @@ -3388,7 +3319,9 @@ def test_does_not_poll_imds_report_ready_when_marker_file_exists( ] m_media_switch.return_value = None dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) - with mock.patch(MOCKPATH + "REPORTED_READY_MARKER_FILE", report_file): + with mock.patch.object( + dsa, "_reported_ready_marker_file", report_file + ): dsa._poll_imds() self.assertEqual(m_report_ready.call_count, 0) @@ -3416,7 +3349,9 @@ def test_poll_imds_report_ready_success_writes_marker_file( m_media_switch.return_value = None dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) self.assertFalse(os.path.exists(report_file)) - with mock.patch(MOCKPATH + "REPORTED_READY_MARKER_FILE", report_file): + with mock.patch.object( + dsa, "_reported_ready_marker_file", report_file + ): dsa._poll_imds() self.assertEqual(m_report_ready.call_count, 1) self.assertTrue(os.path.exists(report_file)) @@ -3446,7 +3381,9 @@ def test_poll_imds_report_ready_failure_raises_exc_and_doesnt_write_marker( m_report_ready.side_effect = [Exception("fail")] dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) self.assertFalse(os.path.exists(report_file)) - with mock.patch(MOCKPATH + "REPORTED_READY_MARKER_FILE", report_file): + with mock.patch.object( + dsa, "_reported_ready_marker_file", report_file + ): self.assertRaises(InvalidMetaDataException, dsa._poll_imds) self.assertEqual(m_report_ready.call_count, 1) self.assertFalse(os.path.exists(report_file)) @@ -3458,8 +3395,8 @@ def test_poll_imds_report_ready_failure_raises_exc_and_doesnt_write_marker( @mock.patch( "cloudinit.sources.helpers.netlink.wait_for_media_disconnect_connect" ) -@mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network", autospec=True) -@mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") +@mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network", autospec=True) +@mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") @mock.patch("requests.Session.request") class TestAzureDataSourcePreprovisioning(CiTestCase): def setUp(self): @@ -3502,6 +3439,7 @@ def test_poll_imds_returns_ovf_env( method="GET", timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, url=full_url, + stream=False, ) ], ) @@ -3535,8 +3473,7 @@ def test__reprovision_calls__poll_imds( full_url = url.format(host) hostname = "myhost" username = "myuser" - odata = {"HostName": hostname, "UserName": username} - content = construct_valid_ovf_env(data=odata) + content = construct_ovf_env(username=username, hostname=hostname) m_request.return_value = mock.MagicMock( status_code=200, text=content, content=content ) @@ -3554,6 +3491,7 @@ def test__reprovision_calls__poll_imds( method="GET", timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, url=full_url, + stream=False, ), m_request.call_args_list, ) @@ -3624,54 +3562,40 @@ def test_remove_network_scripts_default_removes_stock_scripts( self.assertIn(mock.call(path), calls) -class TestWBIsPlatformViable(CiTestCase): - """White box tests for _is_platform_viable.""" +class TestIsPlatformViable: + @pytest.mark.parametrize( + "tag", + [ + dsaz.ChassisAssetTag.AZURE_CLOUD.value, + ], + ) + def test_true_on_azure_chassis( + self, azure_ds, mock_chassis_asset_tag, tag + ): + mock_chassis_asset_tag.return_value = tag - with_logs = True + assert dsaz.is_platform_viable(None) is True - @mock.patch(MOCKPATH + "dmi.read_dmi_data") - def test_true_on_non_azure_chassis(self, m_read_dmi_data): - """Return True if DMI chassis-asset-tag is AZURE_CHASSIS_ASSET_TAG.""" - m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG - self.assertTrue(dsaz._is_platform_viable("doesnotmatter")) + def test_true_on_azure_ovf_env_in_seed_dir( + self, azure_ds, mock_chassis_asset_tag, tmpdir + ): + mock_chassis_asset_tag.return_value = "notazure" - @mock.patch(MOCKPATH + "os.path.exists") - @mock.patch(MOCKPATH + "dmi.read_dmi_data") - def test_true_on_azure_ovf_env_in_seed_dir(self, m_read_dmi_data, m_exist): - """Return True if ovf-env.xml exists in known seed dirs.""" - # Non-matching Azure chassis-asset-tag - m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG + "X" - - m_exist.return_value = True - self.assertTrue(dsaz._is_platform_viable("/some/seed/dir")) - m_exist.called_once_with("/other/seed/dir") - - def test_false_on_no_matching_azure_criteria(self): - """Report non-azure on unmatched asset tag, ovf-env absent and no dev. - - Return False when the asset tag doesn't match Azure's static - AZURE_CHASSIS_ASSET_TAG, no ovf-env.xml files exist in known seed dirs - and no devices have a label starting with prefix 'rd_rdfe_'. - """ - self.assertFalse( - wrap_and_call( - MOCKPATH, - { - "os.path.exists": False, - # Non-matching Azure chassis-asset-tag - "dmi.read_dmi_data": dsaz.AZURE_CHASSIS_ASSET_TAG + "X", - "subp.which": None, - }, - dsaz._is_platform_viable, - "doesnotmatter", - ) - ) - self.assertIn( - "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( - dsaz.AZURE_CHASSIS_ASSET_TAG + "X" - ), - self.logs.getvalue(), - ) + seed_path = Path(azure_ds.seed_dir, "ovf-env.xml") + seed_path.parent.mkdir(exist_ok=True, parents=True) + seed_path.write_text("") + + assert dsaz.is_platform_viable(seed_path.parent) is True + + def test_false_on_no_matching_azure_criteria( + self, azure_ds, mock_chassis_asset_tag + ): + mock_chassis_asset_tag.return_value = None + + seed_path = Path(azure_ds.seed_dir, "ovf-env.xml") + seed_path.parent.mkdir(exist_ok=True, parents=True) + + assert dsaz.is_platform_viable(seed_path) is False class TestRandomSeed(CiTestCase): @@ -4075,6 +3999,20 @@ def test_will_not_retry_errors( ] +class TestInstanceId: + def test_metadata(self, azure_ds, mock_dmi_read_dmi_data): + azure_ds.metadata = {"instance-id": "test-id"} + + id = azure_ds.get_instance_id() + + assert id == "test-id" + + def test_fallback(self, azure_ds, mock_dmi_read_dmi_data): + id = azure_ds.get_instance_id() + + assert id == "fake-system-uuid" + + class TestProvisioning: @pytest.fixture(autouse=True) def provisioning_setup( @@ -4182,7 +4120,8 @@ def test_no_pps(self): # Verify DMI usage. assert self.mock_dmi_read_dmi_data.mock_calls == [ - mock.call("system-uuid") + mock.call("chassis-asset-tag"), + mock.call("system-uuid"), ] assert self.azure_ds.metadata["instance-id"] == "fake-system-uuid" @@ -4203,15 +4142,12 @@ def test_no_pps(self): def test_running_pps(self): self.imds_md["extended"]["compute"]["ppsType"] = "Running" - ovf_data = {"HostName": "myhost", "UserName": "myuser"} nl_sock = mock.MagicMock() self.mock_netlink.create_bound_netlink_socket.return_value = nl_sock self.mock_readurl.side_effect = [ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), - mock.MagicMock( - contents=construct_valid_ovf_env(data=ovf_data).encode() - ), + mock.MagicMock(contents=construct_ovf_env().encode()), mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ] self.mock_azure_get_metadata_from_fabric.return_value = [] @@ -4262,7 +4198,8 @@ def test_running_pps(self): # Verify DMI usage. assert self.mock_dmi_read_dmi_data.mock_calls == [ - mock.call("system-uuid") + mock.call("chassis-asset-tag"), + mock.call("system-uuid"), ] assert self.azure_ds.metadata["instance-id"] == "fake-system-uuid" @@ -4293,7 +4230,6 @@ def test_running_pps(self): def test_savable_pps(self): self.imds_md["extended"]["compute"]["ppsType"] = "Savable" - ovf_data = {"HostName": "myhost", "UserName": "myuser"} nl_sock = mock.MagicMock() self.mock_netlink.create_bound_netlink_socket.return_value = nl_sock @@ -4306,12 +4242,150 @@ def test_savable_pps(self): mock.MagicMock( contents=json.dumps(self.imds_md["network"]).encode() ), + mock.MagicMock(contents=construct_ovf_env().encode()), + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] + self.mock_azure_get_metadata_from_fabric.return_value = [] + + self.azure_ds._get_data() + + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + timeout=2, + headers={"Metadata": "true"}, + retries=10, + exception_cb=dsaz.imds_readurl_exception_callback, + infinite=False, + ), + mock.call( + "http://169.254.169.254/metadata/instance/network?" + "api-version=2021-08-01", + timeout=2, + headers={"Metadata": "true"}, + retries=0, + exception_cb=mock.ANY, + infinite=True, + ), + mock.call( + "http://169.254.169.254/metadata/reprovisiondata?" + "api-version=2019-06-01", + timeout=2, + headers={"Metadata": "true"}, + exception_cb=mock.ANY, + infinite=True, + log_req_resp=False, + ), + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + timeout=2, + headers={"Metadata": "true"}, + retries=10, + exception_cb=dsaz.imds_readurl_exception_callback, + infinite=False, + ), + ] + + # Verify DHCP is setup twice. + assert self.mock_wrapping_setup_ephemeral_networking.mock_calls == [ + mock.call(timeout_minutes=20), + mock.call(iface="ethAttached1", timeout_minutes=20), + ] + assert self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls == [ + mock.call(None, dsaz.dhcp_log_cb), + mock.call("ethAttached1", dsaz.dhcp_log_cb), + ] + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" + assert self.azure_ds._is_ephemeral_networking_up() is False + + # Verify DMI usage. + assert self.mock_dmi_read_dmi_data.mock_calls == [ + mock.call("chassis-asset-tag"), + mock.call("system-uuid"), + ] + assert self.azure_ds.metadata["instance-id"] == "fake-system-uuid" + + # Verify IMDS metadata. + assert self.azure_ds.metadata["imds"] == self.imds_md + + # Verify reporting ready twice. + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ + mock.call( + endpoint="10.11.12.13", + iso_dev="/dev/sr0", + pubkey_info=None, + ), + mock.call( + endpoint="10.11.12.13", + iso_dev=None, + pubkey_info=None, + ), + ] + + # Verify netlink operations for Savable PPS. + assert self.mock_netlink.mock_calls == [ + mock.call.create_bound_netlink_socket(), + mock.call.wait_for_nic_detach_event(nl_sock), + mock.call.wait_for_nic_attach_event(nl_sock, ["ethAttached1"]), + mock.call.create_bound_netlink_socket().__bool__(), + mock.call.create_bound_netlink_socket().close(), + ] + + @pytest.mark.parametrize( + "fabric_side_effect", + [ + [[], []], + [ + [ + url_helper.UrlError( + requests.ConnectionError( + "Failed to establish a new connection: " + "[Errno 101] Network is unreachable" + ) + ) + ], + [], + ], + [ + [url_helper.UrlError(requests.ReadTimeout("Read timed out"))], + [], + ], + ], + ) + def test_savable_pps_early_unplug(self, fabric_side_effect): + self.imds_md["extended"]["compute"]["ppsType"] = "Savable" + + nl_sock = mock.MagicMock() + self.mock_netlink.create_bound_netlink_socket.return_value = nl_sock + self.mock_netlink.wait_for_nic_detach_event.return_value = "eth9" + self.mock_netlink.wait_for_nic_attach_event.return_value = ( + "ethAttached1" + ) + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), mock.MagicMock( - contents=construct_valid_ovf_env(data=ovf_data).encode() + contents=json.dumps(self.imds_md["network"]).encode() ), + mock.MagicMock(contents=construct_ovf_env().encode()), mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ] - self.mock_azure_get_metadata_from_fabric.return_value = [] + self.mock_azure_get_metadata_from_fabric.side_effect = ( + fabric_side_effect + ) + + # Fake DHCP teardown failure. + ipv4_net = self.mock_net_dhcp_EphemeralIPv4Network + ipv4_net.return_value.__exit__.side_effect = [ + subp.ProcessExecutionError( + cmd=["failed", "cmd"], + stdout="test_stdout", + stderr="test_stderr", + exit_code=4, + ), + None, + ] self.azure_ds._get_data() @@ -4368,7 +4442,8 @@ def test_savable_pps(self): # Verify DMI usage. assert self.mock_dmi_read_dmi_data.mock_calls == [ - mock.call("system-uuid") + mock.call("chassis-asset-tag"), + mock.call("system-uuid"), ] assert self.azure_ds.metadata["instance-id"] == "fake-system-uuid" @@ -4402,13 +4477,10 @@ def test_savable_pps(self): def test_recovery_pps(self, pps_type): self.patched_reported_ready_marker_path.write_text("") self.imds_md["extended"]["compute"]["ppsType"] = pps_type - ovf_data = {"HostName": "myhost", "UserName": "myuser"} self.mock_readurl.side_effect = [ mock.MagicMock(contents=json.dumps(self.imds_md).encode()), - mock.MagicMock( - contents=construct_valid_ovf_env(data=ovf_data).encode() - ), + mock.MagicMock(contents=construct_ovf_env().encode()), mock.MagicMock(contents=json.dumps(self.imds_md).encode()), ] self.mock_azure_get_metadata_from_fabric.return_value = [] @@ -4468,6 +4540,69 @@ def test_recovery_pps(self, pps_type): # Verify no netlink operations for recovering PPS. assert self.mock_netlink.mock_calls == [] + @pytest.mark.parametrize( + "subp_side_effect", + [ + subp.SubpResult("okie dokie", ""), + subp.ProcessExecutionError( + cmd=["failed", "cmd"], + stdout="test_stdout", + stderr="test_stderr", + exit_code=4, + ), + ], + ) + def test_os_disk_pps(self, mock_sleep, subp_side_effect): + self.imds_md["extended"]["compute"]["ppsType"] = "PreprovisionedOSDisk" + + self.mock_subp_subp.side_effect = [subp_side_effect] + self.mock_readurl.side_effect = [ + mock.MagicMock(contents=json.dumps(self.imds_md).encode()), + ] + + self.azure_ds._get_data() + + assert self.mock_readurl.mock_calls == [ + mock.call( + "http://169.254.169.254/metadata/instance?" + "api-version=2021-08-01&extended=true", + timeout=2, + headers={"Metadata": "true"}, + retries=10, + exception_cb=dsaz.imds_readurl_exception_callback, + infinite=False, + ) + ] + + assert self.mock_subp_subp.mock_calls == [] + assert mock_sleep.mock_calls == [mock.call(31536000)] + + # Verify DHCP is setup once. + assert self.mock_wrapping_setup_ephemeral_networking.mock_calls == [ + mock.call(timeout_minutes=20) + ] + assert self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls == [ + mock.call(None, dsaz.dhcp_log_cb) + ] + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" + assert self.azure_ds._is_ephemeral_networking_up() is False + + # Verify reported ready once. + assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ + mock.call( + endpoint="10.11.12.13", + iso_dev="/dev/sr0", + pubkey_info=None, + ) + ] + + # Verify no netlink operations for os disk PPS. + assert self.mock_netlink.mock_calls == [] + + # Ensure no reported ready marker is left behind as the VM's next + # boot will behave like a typical provisioning boot. + assert self.patched_reported_ready_marker_path.exists() is False + class TestValidateIMDSMetadata: @pytest.mark.parametrize( diff --git a/tests/unittests/sources/test_azure_helper.py b/tests/unittests/sources/test_azure_helper.py index 4279dc4f6..ff912beff 100644 --- a/tests/unittests/sources/test_azure_helper.py +++ b/tests/unittests/sources/test_azure_helper.py @@ -8,12 +8,16 @@ from xml.sax.saxutils import escape, unescape import pytest +import requests +from cloudinit import url_helper from cloudinit.sources.helpers import azure as azure_helper from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim from cloudinit.util import load_file from tests.unittests.helpers import CiTestCase, ExitStack, mock +from .test_azure import construct_ovf_env + GOAL_STATE_TEMPLATE = """\ + + 1.0 + + + LinuxProvisioningConfiguration + + + + + + """ + + with pytest.raises(azure_helper.BrokenAzureDataSource) as exc_info: + azure_helper.OvfEnvXml.parse_text(ovf) + + assert ( + str(exc_info.value) + == "Multiple configuration matches in ovf-exml.xml " + "for 'ProvisioningSection' (2)" + ) + + def test_multiple_properties_fails(self): + ovf = """\ + + + + + LinuxProvisioningConfiguration + + test-host + test-host2 + test-user + + + + 1.0 + + + + """ + + with pytest.raises(azure_helper.BrokenAzureDataSource) as exc_info: + azure_helper.OvfEnvXml.parse_text(ovf) + + assert ( + str(exc_info.value) + == "Multiple configuration matches in ovf-exml.xml " + "for 'HostName' (2)" + ) + + def test_non_azure_ovf(self): + ovf = """\ + + """ + + with pytest.raises(azure_helper.NonAzureDataSource) as exc_info: + azure_helper.OvfEnvXml.parse_text(ovf) + + assert ( + str(exc_info.value) + == "Ignoring non-Azure ovf-env.xml: ProvisioningSection not found" + ) + + @pytest.mark.parametrize( + "ovf,error", + [ + ("", "Invalid ovf-env.xml: no element found: line 1, column 0"), + ( + "", + "Invalid ovf-env.xml: not well-formed (invalid token): " + "line 1, column 2", + ), + ("badxml", "Invalid ovf-env.xml: syntax error: line 1, column 0"), + ], + ) + def test_invalid_xml(self, ovf, error): + with pytest.raises(azure_helper.BrokenAzureDataSource) as exc_info: + azure_helper.OvfEnvXml.parse_text(ovf) + + assert str(exc_info.value) == error + + # vi: ts=4 expandtab diff --git a/tests/unittests/sources/test_bigstep.py b/tests/unittests/sources/test_bigstep.py new file mode 100644 index 000000000..148cfa0bf --- /dev/null +++ b/tests/unittests/sources/test_bigstep.py @@ -0,0 +1,46 @@ +import json +import os + +import httpretty +import pytest + +from cloudinit import helpers +from cloudinit.sources import DataSourceBigstep as bigstep +from tests.unittests.helpers import mock + +M_PATH = "cloudinit.sources.DataSourceBigstep." + +IMDS_URL = "http://bigstep.com" +METADATA_BODY = json.dumps( + { + "metadata": "metadata", + "vendordata_raw": "vendordata_raw", + "userdata_raw": "userdata_raw", + } +) + + +class TestBigstep: + @httpretty.activate + @pytest.mark.parametrize("custom_paths", [False, True]) + @mock.patch(M_PATH + "util.load_file", return_value=IMDS_URL) + def test_get_data_honor_cloud_dir(self, m_load_file, custom_paths, tmpdir): + httpretty.register_uri(httpretty.GET, IMDS_URL, body=METADATA_BODY) + + paths = {} + url_file = "/var/lib/cloud/data/seed/bigstep/url" + if custom_paths: + paths = { + "cloud_dir": tmpdir.join("cloud"), + "run_dir": tmpdir, + "templates_dir": tmpdir, + } + url_file = os.path.join( + paths["cloud_dir"], "data", "seed", "bigstep", "url" + ) + + ds = bigstep.DataSourceBigstep( + sys_cfg={}, distro=mock.Mock(), paths=helpers.Paths(paths) + ) + assert ds._get_data() + assert [mock.call(url_file)] == m_load_file.call_args_list diff --git a/tests/unittests/sources/test_cloudsigma.py b/tests/unittests/sources/test_cloudsigma.py index 8cd58c963..b92c37238 100644 --- a/tests/unittests/sources/test_cloudsigma.py +++ b/tests/unittests/sources/test_cloudsigma.py @@ -58,12 +58,14 @@ def setUp(self): def test_get_hostname(self): self.datasource.get_data() - self.assertEqual("test_server", self.datasource.get_hostname()) + self.assertEqual( + "test_server", self.datasource.get_hostname().hostname + ) self.datasource.metadata["name"] = "" - self.assertEqual("65b2fb23", self.datasource.get_hostname()) + self.assertEqual("65b2fb23", self.datasource.get_hostname().hostname) utf8_hostname = b"\xd1\x82\xd0\xb5\xd1\x81\xd1\x82".decode("utf-8") self.datasource.metadata["name"] = utf8_hostname - self.assertEqual("65b2fb23", self.datasource.get_hostname()) + self.assertEqual("65b2fb23", self.datasource.get_hostname().hostname) def test_get_public_ssh_keys(self): self.datasource.get_data() diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index f7c69f910..b37400d31 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -40,6 +40,11 @@ def setUp(self): get_networkd_server_address, ) ) + get_data_server = mock.MagicMock(return_value=None) + self.patches.enter_context( + mock.patch(mod_name + ".get_data_server", get_data_server) + ) + self.tmp = self.tmp_dir() def _set_password_server_response(self, response_string): diff --git a/tests/unittests/sources/test_digitalocean.py b/tests/unittests/sources/test_digitalocean.py index f3e6224e8..47e46c66b 100644 --- a/tests/unittests/sources/test_digitalocean.py +++ b/tests/unittests/sources/test_digitalocean.py @@ -178,7 +178,7 @@ def test_metadata(self, mock_readmd): self.assertEqual(DO_META.get("vendor_data"), ds.get_vendordata_raw()) self.assertEqual(DO_META.get("region"), ds.availability_zone) self.assertEqual(DO_META.get("droplet_id"), ds.get_instance_id()) - self.assertEqual(DO_META.get("hostname"), ds.get_hostname()) + self.assertEqual(DO_META.get("hostname"), ds.get_hostname().hostname) # Single key self.assertEqual( diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index bfd7baaa6..b2e629a7d 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -211,7 +211,7 @@ M_PATH_NET = "cloudinit.sources.DataSourceEc2.net." -TAGS_METADATA_2021_03_23 = { +TAGS_METADATA_2021_03_23: dict = { **DEFAULT_METADATA, "tags": { "instance": { @@ -548,7 +548,7 @@ def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp): ): del responses.mock._urls[index] elif hasattr(responses.mock, "_matches"): - # Can be removed when Focal and Impish are EOL + # Can be removed when Focal is EOL for index, response in enumerate(responses.mock._matches): if response.url.startswith( "http://169.254.169.254/2009-04-04/meta-data/" @@ -837,13 +837,14 @@ def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd): self.logs.getvalue(), ) - @mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv6Network") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") @mock.patch("cloudinit.net.find_fallback_nic") - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD") @responses.activate def test_ec2_local_performs_dhcp_on_non_bsd( - self, m_is_bsd, m_dhcp, m_fallback_nic, m_net + self, m_is_bsd, m_dhcp, m_fallback_nic, m_net4, m_net6 ): """Ec2Local returns True for valid platform data on non-BSD with dhcp. @@ -873,7 +874,7 @@ def test_ec2_local_performs_dhcp_on_non_bsd( ret = ds.get_data() self.assertTrue(ret) m_dhcp.assert_called_once_with("eth9", None) - m_net.assert_called_once_with( + m_net4.assert_called_once_with( broadcast="192.168.2.255", interface="eth9", ip="192.168.2.9", @@ -881,7 +882,7 @@ def test_ec2_local_performs_dhcp_on_non_bsd( router="192.168.2.1", static_routes=None, ) - self.assertIn("Crawl of metadata service took", self.logs.getvalue()) + self.assertIn("Crawl of metadata service ", self.logs.getvalue()) @responses.activate def test_get_instance_tags(self): diff --git a/tests/unittests/sources/test_gce.py b/tests/unittests/sources/test_gce.py index e030931b6..1ce0c6ec5 100644 --- a/tests/unittests/sources/test_gce.py +++ b/tests/unittests/sources/test_gce.py @@ -126,7 +126,7 @@ def test_metadata(self): self.ds.get_data() shostname = GCE_META.get("instance/hostname").split(".")[0] - self.assertEqual(shostname, self.ds.get_hostname()) + self.assertEqual(shostname, self.ds.get_hostname().hostname) self.assertEqual( GCE_META.get("instance/id"), self.ds.get_instance_id() @@ -147,7 +147,7 @@ def test_metadata_partial(self): ) shostname = GCE_META_PARTIAL.get("instance/hostname").split(".")[0] - self.assertEqual(shostname, self.ds.get_hostname()) + self.assertEqual(shostname, self.ds.get_hostname().hostname) def test_userdata_no_encoding(self): """check that user-data is read.""" diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index f80ed45f7..193b7e422 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -116,7 +116,7 @@ def test_read_data( self.assertTrue(m_readmd.called) - self.assertEqual(METADATA.get("hostname"), ds.get_hostname()) + self.assertEqual(METADATA.get("hostname"), ds.get_hostname().hostname) self.assertEqual(METADATA.get("public-keys"), ds.get_public_ssh_keys()) diff --git a/tests/unittests/sources/test_init.py b/tests/unittests/sources/test_init.py index ce8fc9700..52f6cbfc9 100644 --- a/tests/unittests/sources/test_init.py +++ b/tests/unittests/sources/test_init.py @@ -17,6 +17,7 @@ UNSET, DataSource, canonical_cloud_id, + pkl_load, redact_sensitive_keys, ) from cloudinit.user_data import UserDataProcessor @@ -272,9 +273,11 @@ def test_get_hostname_strips_local_hostname_without_domain(self): self.assertEqual( "test-subclass-hostname", datasource.metadata["local-hostname"] ) - self.assertEqual("test-subclass-hostname", datasource.get_hostname()) + self.assertEqual( + "test-subclass-hostname", datasource.get_hostname().hostname + ) datasource.metadata["local-hostname"] = "hostname.my.domain.com" - self.assertEqual("hostname", datasource.get_hostname()) + self.assertEqual("hostname", datasource.get_hostname().hostname) def test_get_hostname_with_fqdn_returns_local_hostname_with_domain(self): """Datasource.get_hostname with fqdn set gets qualified hostname.""" @@ -285,7 +288,8 @@ def test_get_hostname_with_fqdn_returns_local_hostname_with_domain(self): self.assertTrue(datasource.get_data()) datasource.metadata["local-hostname"] = "hostname.my.domain.com" self.assertEqual( - "hostname.my.domain.com", datasource.get_hostname(fqdn=True) + "hostname.my.domain.com", + datasource.get_hostname(fqdn=True).hostname, ) def test_get_hostname_without_metadata_uses_system_hostname(self): @@ -300,10 +304,12 @@ def test_get_hostname_without_metadata_uses_system_hostname(self): with mock.patch(mock_fqdn) as m_fqdn: m_gethost.return_value = "systemhostname.domain.com" m_fqdn.return_value = None # No maching fqdn in /etc/hosts - self.assertEqual("systemhostname", datasource.get_hostname()) + self.assertEqual( + "systemhostname", datasource.get_hostname().hostname + ) self.assertEqual( "systemhostname.domain.com", - datasource.get_hostname(fqdn=True), + datasource.get_hostname(fqdn=True).hostname, ) def test_get_hostname_without_metadata_returns_none(self): @@ -316,9 +322,13 @@ def test_get_hostname_without_metadata_returns_none(self): mock_fqdn = "cloudinit.sources.util.get_fqdn_from_hosts" with mock.patch("cloudinit.sources.util.get_hostname") as m_gethost: with mock.patch(mock_fqdn) as m_fqdn: - self.assertIsNone(datasource.get_hostname(metadata_only=True)) self.assertIsNone( - datasource.get_hostname(fqdn=True, metadata_only=True) + datasource.get_hostname(metadata_only=True).hostname + ) + self.assertIsNone( + datasource.get_hostname( + fqdn=True, metadata_only=True + ).hostname ) self.assertEqual([], m_gethost.call_args_list) self.assertEqual([], m_fqdn.call_args_list) @@ -335,10 +345,12 @@ def test_get_hostname_without_metadata_prefers_etc_hosts(self): with mock.patch(mock_fqdn) as m_fqdn: m_gethost.return_value = "systemhostname.domain.com" m_fqdn.return_value = "fqdnhostname.domain.com" - self.assertEqual("fqdnhostname", datasource.get_hostname()) + self.assertEqual( + "fqdnhostname", datasource.get_hostname().hostname + ) self.assertEqual( "fqdnhostname.domain.com", - datasource.get_hostname(fqdn=True), + datasource.get_hostname(fqdn=True).hostname, ) def test_get_data_does_not_write_instance_data_on_failure(self): @@ -661,8 +673,12 @@ def test_get_data_handles_redacted_unserializable_content(self): def test_persist_instance_data_writes_ec2_metadata_when_set(self): """When ec2_metadata class attribute is set, persist to json.""" tmp = self.tmp_dir() + cloud_dir = os.path.join(tmp, "cloud") + util.ensure_dir(cloud_dir) datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) + self.sys_cfg, + self.distro, + Paths({"run_dir": tmp, "cloud_dir": cloud_dir}), ) datasource.ec2_metadata = UNSET datasource.get_data() @@ -679,8 +695,12 @@ def test_persist_instance_data_writes_ec2_metadata_when_set(self): def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): """canonical-cloud-id class attribute is set, persist to json.""" tmp = self.tmp_dir() + cloud_dir = os.path.join(tmp, "cloud") + util.ensure_dir(cloud_dir) datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) + self.sys_cfg, + self.distro, + Paths({"run_dir": tmp, "cloud_dir": cloud_dir}), ) cloud_id_link = os.path.join(tmp, "cloud-id") cloud_id_file = os.path.join(tmp, "cloud-id-my-cloud") @@ -711,8 +731,12 @@ def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): def test_persist_instance_data_writes_network_json_when_set(self): """When network_data.json class attribute is set, persist to json.""" tmp = self.tmp_dir() + cloud_dir = os.path.join(tmp, "cloud") + util.ensure_dir(cloud_dir) datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) + self.sys_cfg, + self.distro, + Paths({"run_dir": tmp, "cloud_dir": cloud_dir}), ) datasource.get_data() json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) @@ -725,6 +749,34 @@ def test_persist_instance_data_writes_network_json_when_set(self): {"network_json": "is good"}, instance_data["ds"]["network_json"] ) + def test_persist_instance_serializes_datasource_pickle(self): + """obj.pkl is written when instance link present and write_cache.""" + tmp = self.tmp_dir() + cloud_dir = os.path.join(tmp, "cloud") + util.ensure_dir(cloud_dir) + datasource = DataSourceTestSubclassNet( + self.sys_cfg, + self.distro, + Paths({"run_dir": tmp, "cloud_dir": cloud_dir}), + ) + pkl_cache_file = os.path.join(cloud_dir, "instance/obj.pkl") + self.assertFalse(os.path.exists(pkl_cache_file)) + datasource.network_json = {"network_json": "is good"} + # No /var/lib/cloud/instance symlink + datasource.persist_instance_data(write_cache=True) + self.assertFalse(os.path.exists(pkl_cache_file)) + + # Symlink /var/lib/cloud/instance but write_cache=False + util.sym_link(cloud_dir, os.path.join(cloud_dir, "instance")) + datasource.persist_instance_data(write_cache=False) + self.assertFalse(os.path.exists(pkl_cache_file)) + + # Symlink /var/lib/cloud/instance and write_cache=True + datasource.persist_instance_data(write_cache=True) + self.assertTrue(os.path.exists(pkl_cache_file)) + ds = pkl_load(pkl_cache_file) + self.assertEqual(datasource.network_json, ds.network_json) + def test_get_data_base64encodes_unserializable_bytes(self): """On py3, get_data base64encodes any unserializable content.""" tmp = self.tmp_dir() @@ -750,7 +802,9 @@ def test_get_hostname_subclass_support(self): """Validate get_hostname signature on all subclasses of DataSource.""" base_args = inspect.getfullargspec(DataSource.get_hostname) # Import all DataSource subclasses so we can inspect them. - modules = util.find_modules(os.path.dirname(os.path.dirname(__file__))) + modules = util.get_modules_from_dir( + os.path.dirname(os.path.dirname(__file__)) + ) for _loc, name in modules.items(): mod_locs, _ = importer.find_module(name, ["cloudinit.sources"], []) if mod_locs: diff --git a/tests/unittests/sources/test_lxd.py b/tests/unittests/sources/test_lxd.py index e11c3746d..e60bb71fa 100644 --- a/tests/unittests/sources/test_lxd.py +++ b/tests/unittests/sources/test_lxd.py @@ -17,7 +17,7 @@ DS_PATH = "cloudinit.sources.DataSourceLXD." -LStatResponse = namedtuple("lstatresponse", "st_mode") +LStatResponse = namedtuple("LStatResponse", "st_mode") NETWORK_V1 = { @@ -34,7 +34,7 @@ def _add_network_v1_device(devname) -> dict: """Helper to inject device name into default network v1 config.""" - network_cfg = deepcopy(NETWORK_V1) + network_cfg: dict = deepcopy(NETWORK_V1) network_cfg["config"][0]["name"] = devname return network_cfg @@ -51,14 +51,27 @@ def _add_network_v1_device(devname) -> dict: }, } +LXD_V1_METADATA_NO_NETWORK_CONFIG = { + "meta-data": "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n", + "user-data": "#cloud-config\npackages: [sl]\n", + "vendor-data": "#cloud-config\nruncmd: ['echo vendor-data']\n", + "config": { + "user.user-data": "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n", + "user.vendor-data": "#cloud-config\nruncmd: ['echo vendor-data']\n", + }, +} + -@pytest.fixture def lxd_metadata(): return LXD_V1_METADATA +def lxd_metadata_no_network_config(): + return LXD_V1_METADATA_NO_NETWORK_CONFIG + + @pytest.fixture -def lxd_ds(request, paths, lxd_metadata): +def lxd_ds(request, paths): """ Return an instantiated DataSourceLXD. @@ -69,7 +82,30 @@ def lxd_ds(request, paths, lxd_metadata): (This uses the paths fixture for the required helpers.Paths object) """ with mock.patch(DS_PATH + "is_platform_viable", return_value=True): - with mock.patch(DS_PATH + "read_metadata", return_value=lxd_metadata): + with mock.patch( + DS_PATH + "read_metadata", return_value=lxd_metadata() + ): + yield lxd.DataSourceLXD( + sys_cfg={}, distro=mock.Mock(), paths=paths + ) + + +@pytest.fixture +def lxd_ds_no_network_config(request, paths): + """ + Return an instantiated DataSourceLXD. + + This also performs the mocking required for the default test case: + * ``is_platform_viable`` returns True, + * ``read_metadata`` returns ``LXD_V1_METADATA_NO_NETWORK_CONFIG`` + + (This uses the paths fixture for the required helpers.Paths object) + """ + with mock.patch(DS_PATH + "is_platform_viable", return_value=True): + with mock.patch( + DS_PATH + "read_metadata", + return_value=lxd_metadata_no_network_config(), + ): yield lxd.DataSourceLXD( sys_cfg={}, distro=mock.Mock(), paths=paths ) @@ -142,6 +178,37 @@ def test__get_data(self, lxd_ds): assert LXD_V1_METADATA["user-data"] == lxd_ds.userdata_raw assert LXD_V1_METADATA["vendor-data"] == lxd_ds.vendordata_raw + def test_network_config_when_unset(self, lxd_ds): + """network_config is correctly computed when _network_config and + _crawled_metadata are unset. + """ + assert UNSET == lxd_ds._crawled_metadata + assert UNSET == lxd_ds._network_config + assert None is lxd_ds.userdata_raw + # network-config is dumped from YAML + assert NETWORK_V1 == lxd_ds.network_config + assert LXD_V1_METADATA == lxd_ds._crawled_metadata + + def test_network_config_crawled_metadata_no_network_config( + self, lxd_ds_no_network_config + ): + """network_config is correctly computed when _network_config is unset + and _crawled_metadata does not contain network_config. + """ + lxd.generate_fallback_network_config = mock.Mock( + return_value=NETWORK_V1 + ) + assert UNSET == lxd_ds_no_network_config._crawled_metadata + assert UNSET == lxd_ds_no_network_config._network_config + assert None is lxd_ds_no_network_config.userdata_raw + # network-config is dumped from YAML + assert NETWORK_V1 == lxd_ds_no_network_config.network_config + assert ( + LXD_V1_METADATA_NO_NETWORK_CONFIG + == lxd_ds_no_network_config._crawled_metadata + ) + assert 1 == lxd.generate_fallback_network_config.call_count + class TestIsPlatformViable: @pytest.mark.parametrize( diff --git a/tests/unittests/sources/test_opennebula.py b/tests/unittests/sources/test_opennebula.py index e05c47491..af1c45b81 100644 --- a/tests/unittests/sources/test_opennebula.py +++ b/tests/unittests/sources/test_opennebula.py @@ -73,7 +73,7 @@ def test_get_data_non_contextdisk(self): orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - util.find_devs_with = lambda n: [] + util.find_devs_with = lambda n: [] # type: ignore dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() self.assertFalse(ret) @@ -84,7 +84,7 @@ def test_get_data_broken_contextdisk(self): orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - util.find_devs_with = lambda n: [] + util.find_devs_with = lambda n: [] # type: ignore populate_dir(self.seed_dir, {"context.sh": INVALID_CONTEXT}) dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) @@ -107,7 +107,7 @@ def test_get_data_invalid_identity(self): ] = invalid_user # dont' try to lookup for CDs - util.find_devs_with = lambda n: [] + util.find_devs_with = lambda n: [] # type: ignore populate_context_dir(self.seed_dir, {"KEY1": "val1"}) dsrc = self.ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) self.assertRaises(ds.BrokenContextDiskDir, dsrc.get_data) @@ -118,7 +118,7 @@ def test_get_data(self): orig_find_devs_with = util.find_devs_with try: # dont' try to lookup for CDs - util.find_devs_with = lambda n: [] + util.find_devs_with = lambda n: [] # type: ignore populate_context_dir(self.seed_dir, {"KEY1": "val1"}) dsrc = self.ds(sys_cfg=self.sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() diff --git a/tests/unittests/sources/test_openstack.py b/tests/unittests/sources/test_openstack.py index c111bbcd1..f65aab8b8 100644 --- a/tests/unittests/sources/test_openstack.py +++ b/tests/unittests/sources/test_openstack.py @@ -39,7 +39,7 @@ VENDOR_DATA = { "magic": "", } -VENDOR_DATA2 = {"static": {}} +VENDOR_DATA2: dict = {"static": {}} OSTACK_META = { "availability_zone": "nova", "files": [ @@ -284,8 +284,10 @@ def test_datasource(self, m_dhcp): m_dhcp.assert_not_called() @hp.activate - @test_helpers.mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network") - @test_helpers.mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") + @test_helpers.mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") + @test_helpers.mock.patch( + "cloudinit.net.ephemeral.maybe_perform_dhcp_discovery" + ) def test_local_datasource(self, m_dhcp, m_net): """OpenStackLocal calls EphemeralDHCPNetwork and gets instance data.""" _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) diff --git a/tests/unittests/sources/test_oracle.py b/tests/unittests/sources/test_oracle.py index c652b6d16..b5b195be8 100644 --- a/tests/unittests/sources/test_oracle.py +++ b/tests/unittests/sources/test_oracle.py @@ -3,7 +3,7 @@ import base64 import copy import json -from contextlib import ExitStack +import logging from unittest import mock import pytest @@ -13,6 +13,7 @@ from cloudinit.sources.DataSourceOracle import OpcMetadata from cloudinit.url_helper import UrlError from tests.unittests import helpers as test_helpers +from tests.unittests.helpers import does_not_raise DS_PATH = "cloudinit.sources.DataSourceOracle" @@ -87,6 +88,25 @@ # Just a small meaningless change to differentiate the two metadatas OPC_V1_METADATA = OPC_V2_METADATA.replace("ocid1.instance", "ocid2.instance") +MAC_ADDR = "00:00:17:02:2b:b1" + +DHCP = { + "name": "eth0", + "type": "physical", + "subnets": [ + { + "broadcast": "192.168.122.255", + "control": "manual", + "gateway": "192.168.122.1", + "dns_search": ["foo.com"], + "type": "dhcp", + "netmask": "255.255.255.0", + "dns_nameservers": ["192.168.122.1"], + } + ], +} +KLIBC_NET_CFG = {"version": 1, "config": [DHCP]} + @pytest.fixture def metadata_version(): @@ -94,15 +114,20 @@ def metadata_version(): @pytest.fixture -def oracle_ds(request, fixture_utils, paths, metadata_version): +def oracle_ds(request, fixture_utils, paths, metadata_version, mocker): """ Return an instantiated DataSourceOracle. - This also performs the mocking required for the default test case: + This also performs the mocking required: * ``_read_system_uuid`` returns something, * ``_is_platform_viable`` returns True, - * ``_is_iscsi_root`` returns True (the simpler code path), - * ``read_opc_metadata`` returns ``OPC_V1_METADATA`` + * ``DataSourceOracle._is_iscsi_root`` returns True by default or what + pytest.mark.is_iscsi gives as first param, + * ``DataSourceOracle._get_iscsi_config`` returns a network cfg if + is_iscsi else an empty network config, + * ``read_opc_metadata`` returns ``OPC_V1_METADATA``, + * ``ephemeral.EphemeralDHCPv4`` and ``net.find_fallback_nic`` mocked to + avoid subp calls (This uses the paths fixture for the required helpers.Paths object, and the fixture_utils fixture for fetching markers.) @@ -110,19 +135,29 @@ def oracle_ds(request, fixture_utils, paths, metadata_version): sys_cfg = fixture_utils.closest_marker_first_arg_or( request, "ds_sys_cfg", mock.MagicMock() ) + is_iscsi = fixture_utils.closest_marker_first_arg_or( + request, "is_iscsi", True + ) metadata = OpcMetadata(metadata_version, json.loads(OPC_V2_METADATA), None) - with mock.patch(DS_PATH + "._read_system_uuid", return_value="someuuid"): - with mock.patch(DS_PATH + "._is_platform_viable", return_value=True): - with mock.patch(DS_PATH + "._is_iscsi_root", return_value=True): - with mock.patch( - DS_PATH + ".read_opc_metadata", - return_value=metadata, - ): - yield oracle.DataSourceOracle( - sys_cfg=sys_cfg, - distro=mock.Mock(), - paths=paths, - ) + + mocker.patch(DS_PATH + ".net.find_fallback_nic") + mocker.patch(DS_PATH + ".ephemeral.EphemeralDHCPv4") + mocker.patch(DS_PATH + "._read_system_uuid", return_value="someuuid") + mocker.patch(DS_PATH + "._is_platform_viable", return_value=True) + mocker.patch(DS_PATH + ".read_opc_metadata", return_value=metadata) + mocker.patch(DS_PATH + ".KlibcOracleNetworkConfigSource") + ds = oracle.DataSourceOracle( + sys_cfg=sys_cfg, + distro=mock.Mock(), + paths=paths, + ) + mocker.patch.object(ds, "_is_iscsi_root", return_value=is_iscsi) + if is_iscsi: + iscsi_config = copy.deepcopy(KLIBC_NET_CFG) + else: + iscsi_config = {"version": 1, "config": []} + mocker.patch.object(ds, "_get_iscsi_config", return_value=iscsi_config) + yield ds class TestDataSourceOracle: @@ -158,28 +193,27 @@ def test_sys_cfg_can_enable_configure_secondary_nics(self, oracle_ds): assert oracle_ds.ds_cfg["configure_secondary_nics"] -class TestIsPlatformViable(test_helpers.CiTestCase): - @mock.patch( - DS_PATH + ".dmi.read_dmi_data", return_value=oracle.CHASSIS_ASSET_TAG +class TestIsPlatformViable: + @pytest.mark.parametrize( + "dmi_data, platform_viable", + [ + # System with known chassis tag is viable. + (oracle.CHASSIS_ASSET_TAG, True), + # System without known chassis tag is not viable. + (None, False), + # System with unknown chassis tag is not viable. + ("LetsGoCubs", False), + ], ) - def test_expected_viable(self, m_read_dmi_data): - """System with known chassis tag is viable.""" - self.assertTrue(oracle._is_platform_viable()) - m_read_dmi_data.assert_has_calls([mock.call("chassis-asset-tag")]) - - @mock.patch(DS_PATH + ".dmi.read_dmi_data", return_value=None) - def test_expected_not_viable_dmi_data_none(self, m_read_dmi_data): - """System without known chassis tag is not viable.""" - self.assertFalse(oracle._is_platform_viable()) - m_read_dmi_data.assert_has_calls([mock.call("chassis-asset-tag")]) - - @mock.patch(DS_PATH + ".dmi.read_dmi_data", return_value="LetsGoCubs") - def test_expected_not_viable_other(self, m_read_dmi_data): - """System with unnown chassis tag is not viable.""" - self.assertFalse(oracle._is_platform_viable()) + def test_is_platform_viable(self, dmi_data, platform_viable): + with mock.patch( + DS_PATH + ".dmi.read_dmi_data", return_value=dmi_data + ) as m_read_dmi_data: + assert platform_viable == oracle._is_platform_viable() m_read_dmi_data.assert_has_calls([mock.call("chassis-asset-tag")]) +@pytest.mark.is_iscsi(False) @mock.patch( "cloudinit.net.is_openvswitch_internal_interface", mock.Mock(return_value=False), @@ -190,7 +224,7 @@ def test_no_secondary_nics_does_not_mutate_input(self, oracle_ds): # We test this by using in a non-dict to ensure that no dict # operations are used; failure would be seen as exceptions oracle_ds._network_config = object() - oracle_ds._add_network_config_from_opc_imds() + oracle_ds._add_network_config_from_opc_imds(set_primary=False) def test_bare_metal_machine_skipped(self, oracle_ds, caplog): # nicIndex in the first entry indicates a bare metal machine @@ -198,165 +232,307 @@ def test_bare_metal_machine_skipped(self, oracle_ds, caplog): # We test this by using a non-dict to ensure that no dict # operations are used oracle_ds._network_config = object() - oracle_ds._add_network_config_from_opc_imds() + oracle_ds._add_network_config_from_opc_imds(set_primary=False) assert "bare metal machine" in caplog.text - def test_missing_mac_skipped(self, oracle_ds, caplog): - oracle_ds._vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) - - oracle_ds._network_config = { - "version": 1, - "config": [{"primary": "nic"}], - } - with mock.patch(DS_PATH + ".get_interfaces_by_mac", return_value={}): - oracle_ds._add_network_config_from_opc_imds() - - assert 1 == len(oracle_ds.network_config["config"]) - assert ( - "Interface with MAC 00:00:17:02:2b:b1 not found; skipping" - in caplog.text - ) - - def test_missing_mac_skipped_v2(self, oracle_ds, caplog): + @pytest.mark.parametrize( + "network_config, network_config_key", + [ + pytest.param( + { + "version": 1, + "config": [{"primary": "nic"}], + }, + "config", + id="v1", + ), + pytest.param( + { + "version": 2, + "ethernets": {"primary": {"nic": {}}}, + }, + "ethernets", + id="v2", + ), + ], + ) + def test_missing_mac_skipped( + self, + oracle_ds, + network_config, + network_config_key, + caplog, + ): oracle_ds._vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) - - oracle_ds._network_config = { - "version": 2, - "ethernets": {"primary": {"nic": {}}}, - } + oracle_ds._network_config = network_config with mock.patch(DS_PATH + ".get_interfaces_by_mac", return_value={}): - oracle_ds._add_network_config_from_opc_imds() + oracle_ds._add_network_config_from_opc_imds(set_primary=False) - assert 1 == len(oracle_ds.network_config["ethernets"]) + assert 1 == len(oracle_ds._network_config[network_config_key]) assert ( - "Interface with MAC 00:00:17:02:2b:b1 not found; skipping" - in caplog.text + f"Interface with MAC {MAC_ADDR} not found; skipping" in caplog.text ) + assert 1 == caplog.text.count(" not found; skipping") - def test_secondary_nic(self, oracle_ds): + @pytest.mark.parametrize( + "set_primary", + [True, False], + ) + def test_imds_nic_setup_v1(self, set_primary, oracle_ds): oracle_ds._vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) oracle_ds._network_config = { "version": 1, "config": [{"primary": "nic"}], } - mac_addr, nic_name = "00:00:17:02:2b:b1", "ens3" with mock.patch( - DS_PATH + ".get_interfaces_by_mac", - return_value={mac_addr: nic_name}, + f"{DS_PATH}.get_interfaces_by_mac", + return_value={ + "02:00:17:05:d1:db": "ens3", + "00:00:17:02:2b:b1": "ens4", + }, ): - oracle_ds._add_network_config_from_opc_imds() - - # The input is mutated - assert 2 == len(oracle_ds.network_config["config"]) - - secondary_nic_cfg = oracle_ds.network_config["config"][1] - assert nic_name == secondary_nic_cfg["name"] - assert "physical" == secondary_nic_cfg["type"] - assert mac_addr == secondary_nic_cfg["mac_address"] - assert 9000 == secondary_nic_cfg["mtu"] + oracle_ds._add_network_config_from_opc_imds( + set_primary=set_primary + ) - assert 1 == len(secondary_nic_cfg["subnets"]) - subnet_cfg = secondary_nic_cfg["subnets"][0] - # These values are hard-coded in OPC_VM_SECONDARY_VNIC_RESPONSE - assert "10.0.0.231" == subnet_cfg["address"] + secondary_nic_index = 1 + nic_cfg = oracle_ds.network_config["config"] + if set_primary: + primary_cfg = nic_cfg[1] + secondary_nic_index += 1 + + assert "ens3" == primary_cfg["name"] + assert "physical" == primary_cfg["type"] + assert "02:00:17:05:d1:db" == primary_cfg["mac_address"] + assert 9000 == primary_cfg["mtu"] + assert 1 == len(primary_cfg["subnets"]) + assert "address" not in primary_cfg["subnets"][0] + assert "dhcp" == primary_cfg["subnets"][0]["type"] + secondary_cfg = nic_cfg[secondary_nic_index] + assert "ens4" == secondary_cfg["name"] + assert "physical" == secondary_cfg["type"] + assert "00:00:17:02:2b:b1" == secondary_cfg["mac_address"] + assert 9000 == secondary_cfg["mtu"] + assert 1 == len(secondary_cfg["subnets"]) + assert "10.0.0.231/24" == secondary_cfg["subnets"][0]["address"] + assert "static" == secondary_cfg["subnets"][0]["type"] - def test_secondary_nic_v2(self, oracle_ds): + @pytest.mark.parametrize( + "set_primary", + [True, False], + ) + def test_secondary_nic_v2(self, set_primary, oracle_ds): oracle_ds._vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) oracle_ds._network_config = { "version": 2, "ethernets": {"primary": {"nic": {}}}, } - mac_addr, nic_name = "00:00:17:02:2b:b1", "ens3" with mock.patch( - DS_PATH + ".get_interfaces_by_mac", - return_value={mac_addr: nic_name}, + f"{DS_PATH}.get_interfaces_by_mac", + return_value={ + "02:00:17:05:d1:db": "ens3", + "00:00:17:02:2b:b1": "ens4", + }, ): - oracle_ds._add_network_config_from_opc_imds() + oracle_ds._add_network_config_from_opc_imds( + set_primary=set_primary + ) - # The input is mutated - assert 2 == len(oracle_ds.network_config["ethernets"]) + nic_cfg = oracle_ds.network_config["ethernets"] + if set_primary: + assert "ens3" in nic_cfg + primary_cfg = nic_cfg["ens3"] - secondary_nic_cfg = oracle_ds.network_config["ethernets"]["ens3"] - assert secondary_nic_cfg["dhcp4"] is False - assert secondary_nic_cfg["dhcp6"] is False - assert "ens3" == secondary_nic_cfg["match"]["name"] - assert 9000 == secondary_nic_cfg["mtu"] + assert primary_cfg["dhcp4"] is True + assert primary_cfg["dhcp6"] is False + assert "ens3" == primary_cfg["match"]["name"] + assert 9000 == primary_cfg["mtu"] + assert "addresses" not in primary_cfg - assert 1 == len(secondary_nic_cfg["addresses"]) - # These values are hard-coded in OPC_VM_SECONDARY_VNIC_RESPONSE - assert "10.0.0.231" == secondary_nic_cfg["addresses"][0] + assert "ens4" in nic_cfg + secondary_cfg = nic_cfg["ens4"] + assert secondary_cfg["dhcp4"] is False + assert secondary_cfg["dhcp6"] is False + assert "ens4" == secondary_cfg["match"]["name"] + assert 9000 == secondary_cfg["mtu"] + assert 1 == len(secondary_cfg["addresses"]) + assert "10.0.0.231/24" == secondary_cfg["addresses"][0] -class TestNetworkConfigFiltersNetFailover(test_helpers.CiTestCase): - def setUp(self): - super(TestNetworkConfigFiltersNetFailover, self).setUp() - self.add_patch( - DS_PATH + ".get_interfaces_by_mac", "m_get_interfaces_by_mac" - ) - self.add_patch(DS_PATH + ".is_netfail_master", "m_netfail_master") + @pytest.mark.parametrize("error_add_network", [None, Exception]) + @pytest.mark.parametrize( + "configure_secondary_nics", + [False, True], + ) + @mock.patch(DS_PATH + "._ensure_netfailover_safe") + def test_network_config_log_errors( + self, + m_ensure_netfailover_safe, + configure_secondary_nics, + error_add_network, + oracle_ds, + caplog, + capsys, + ): + assert not oracle_ds._has_network_config() + oracle_ds.ds_cfg["configure_secondary_nics"] = configure_secondary_nics + with mock.patch.object( + oracle.DataSourceOracle, + "_add_network_config_from_opc_imds", + ) as m_add_network_config_from_opc_imds: + if error_add_network: + m_add_network_config_from_opc_imds.side_effect = ( + error_add_network + ) + oracle_ds.network_config # pylint: disable=pointless-statement # noqa: E501 + assert [ + mock.call(True, False) + == m_add_network_config_from_opc_imds.call_args_list + ] + assert 1 == oracle_ds._is_iscsi_root.call_count + assert 1 == m_ensure_netfailover_safe.call_count + + assert ("", "") == capsys.readouterr() + if not error_add_network: + log_initramfs_index = -1 + else: + log_initramfs_index = -3 + # Primary + assert ( + logging.WARNING, + "Failed to parse IMDS network configuration!", + ) == caplog.record_tuples[-2][1:] + # Secondary + assert ( + logging.DEBUG, + "Failed to parse IMDS network configuration!", + ) == caplog.record_tuples[-1][1:] - def test_ignore_bogus_network_config(self): - netcfg = {"something": "here"} - passed_netcfg = copy.copy(netcfg) - oracle._ensure_netfailover_safe(passed_netcfg) - self.assertEqual(netcfg, passed_netcfg) + assert ( + logging.WARNING, + "Could not obtain network configuration from initramfs." + " Falling back to IMDS.", + ) == caplog.record_tuples[log_initramfs_index][1:] - def test_ignore_network_config_unknown_versions(self): - netcfg = {"something": "here", "version": 3} - passed_netcfg = copy.copy(netcfg) - oracle._ensure_netfailover_safe(passed_netcfg) - self.assertEqual(netcfg, passed_netcfg) - def test_checks_v1_type_physical_interfaces(self): - mac_addr, nic_name = "00:00:17:02:2b:b1", "ens3" - self.m_get_interfaces_by_mac.return_value = { - mac_addr: nic_name, - } - netcfg = { - "version": 1, - "config": [ - { - "type": "physical", - "name": nic_name, - "mac_address": mac_addr, - "subnets": [{"type": "dhcp4"}], - } - ], - } +@mock.patch(DS_PATH + ".get_interfaces_by_mac") +@mock.patch(DS_PATH + ".is_netfail_master") +class TestNetworkConfigFiltersNetFailover: + @pytest.mark.parametrize( + "netcfg", + [ + pytest.param({"something": "here"}, id="bogus"), + pytest.param( + {"something": "here", "version": 3}, id="unknown_version" + ), + ], + ) + def test_ignore_network_config( + self, m_netfail_master, m_get_interfaces_by_mac, netcfg + ): passed_netcfg = copy.copy(netcfg) - self.m_netfail_master.return_value = False oracle._ensure_netfailover_safe(passed_netcfg) - self.assertEqual(netcfg, passed_netcfg) - self.assertEqual( - [mock.call(nic_name)], self.m_netfail_master.call_args_list - ) + assert netcfg == passed_netcfg - def test_checks_v1_skips_non_phys_interfaces(self): - mac_addr, nic_name = "00:00:17:02:2b:b1", "bond0" - self.m_get_interfaces_by_mac.return_value = { - mac_addr: nic_name, - } - netcfg = { - "version": 1, - "config": [ + @pytest.mark.parametrize( + "nic_name, netcfg, netfail_master_return, call_args_list", + [ + pytest.param( + "ens3", { - "type": "bond", - "name": nic_name, - "mac_address": mac_addr, - "subnets": [{"type": "dhcp4"}], - } - ], + "version": 1, + "config": [ + { + "type": "physical", + "name": "ens3", + "mac_address": MAC_ADDR, + "subnets": [{"type": "dhcp4"}], + } + ], + }, + False, + [mock.call("ens3")], + id="checks_v1_type_physical_interfaces", + ), + pytest.param( + "bond0", + { + "version": 1, + "config": [ + { + "type": "bond", + "name": "bond0", + "mac_address": MAC_ADDR, + "subnets": [{"type": "dhcp4"}], + } + ], + }, + None, + [], + id="skips_v1_non_phys_interfaces", + ), + pytest.param( + "ens3", + { + "version": 2, + "ethernets": { + "ens3": { + "dhcp4": True, + "critical": True, + "set-name": "ens3", + "match": {"macaddress": MAC_ADDR}, + } + }, + }, + False, + [mock.call("ens3")], + id="checks_v2_type_ethernet_interfaces", + ), + pytest.param( + "wlps0", + { + "version": 2, + "ethernets": { + "wlps0": { + "dhcp4": True, + "critical": True, + "set-name": "wlps0", + "match": {"macaddress": MAC_ADDR}, + } + }, + }, + None, + [mock.call("wlps0")], + id="skips_v2_non_ethernet_interfaces", + ), + ], + ) + def test__ensure_netfailover_safe( + self, + m_netfail_master, + m_get_interfaces_by_mac, + nic_name, + netcfg, + netfail_master_return, + call_args_list, + ): + m_get_interfaces_by_mac.return_value = { + MAC_ADDR: nic_name, } passed_netcfg = copy.copy(netcfg) + if netfail_master_return is not None: + m_netfail_master.return_value = netfail_master_return oracle._ensure_netfailover_safe(passed_netcfg) - self.assertEqual(netcfg, passed_netcfg) - self.assertEqual(0, self.m_netfail_master.call_count) - - def test_removes_master_mac_property_v1(self): - nic_master, mac_master = "ens3", self.random_string() - nic_other, mac_other = "ens7", self.random_string() - nic_extra, mac_extra = "enp0s1f2", self.random_string() - self.m_get_interfaces_by_mac.return_value = { + assert netcfg == passed_netcfg + assert call_args_list == m_netfail_master.call_args_list + + def test_removes_master_mac_property_v1( + self, m_netfail_master, m_get_interfaces_by_mac + ): + nic_master, mac_master = "ens3", test_helpers.random_string() + nic_other, mac_other = "ens7", test_helpers.random_string() + nic_extra, mac_extra = "enp0s1f2", test_helpers.random_string() + m_get_interfaces_by_mac.return_value = { mac_master: nic_master, mac_other: nic_other, mac_extra: nic_extra, @@ -387,7 +563,7 @@ def _is_netfail_master(iface): return True return False - self.m_netfail_master.side_effect = _is_netfail_master + m_netfail_master.side_effect = _is_netfail_master expected_cfg = { "version": 1, "config": [ @@ -405,58 +581,15 @@ def _is_netfail_master(iface): ], } oracle._ensure_netfailover_safe(netcfg) - self.assertEqual(expected_cfg, netcfg) + assert expected_cfg == netcfg - def test_checks_v2_type_ethernet_interfaces(self): - mac_addr, nic_name = "00:00:17:02:2b:b1", "ens3" - self.m_get_interfaces_by_mac.return_value = { - mac_addr: nic_name, - } - netcfg = { - "version": 2, - "ethernets": { - nic_name: { - "dhcp4": True, - "critical": True, - "set-name": nic_name, - "match": {"macaddress": mac_addr}, - } - }, - } - passed_netcfg = copy.copy(netcfg) - self.m_netfail_master.return_value = False - oracle._ensure_netfailover_safe(passed_netcfg) - self.assertEqual(netcfg, passed_netcfg) - self.assertEqual( - [mock.call(nic_name)], self.m_netfail_master.call_args_list - ) - - def test_skips_v2_non_ethernet_interfaces(self): - mac_addr, nic_name = "00:00:17:02:2b:b1", "wlps0" - self.m_get_interfaces_by_mac.return_value = { - mac_addr: nic_name, - } - netcfg = { - "version": 2, - "wifis": { - nic_name: { - "dhcp4": True, - "critical": True, - "set-name": nic_name, - "match": {"macaddress": mac_addr}, - } - }, - } - passed_netcfg = copy.copy(netcfg) - oracle._ensure_netfailover_safe(passed_netcfg) - self.assertEqual(netcfg, passed_netcfg) - self.assertEqual(0, self.m_netfail_master.call_count) - - def test_removes_master_mac_property_v2(self): - nic_master, mac_master = "ens3", self.random_string() - nic_other, mac_other = "ens7", self.random_string() - nic_extra, mac_extra = "enp0s1f2", self.random_string() - self.m_get_interfaces_by_mac.return_value = { + def test_removes_master_mac_property_v2( + self, m_netfail_master, m_get_interfaces_by_mac + ): + nic_master, mac_master = "ens3", test_helpers.random_string() + nic_other, mac_other = "ens7", test_helpers.random_string() + nic_extra, mac_extra = "enp0s1f2", test_helpers.random_string() + m_get_interfaces_by_mac.return_value = { mac_master: nic_master, mac_other: nic_other, mac_extra: nic_extra, @@ -487,7 +620,7 @@ def _is_netfail_master(iface): return True return False - self.m_netfail_master.side_effect = _is_netfail_master + m_netfail_master.side_effect = _is_netfail_master expected_cfg = { "version": 2, @@ -511,7 +644,7 @@ def _is_netfail_master(iface): pprint.pprint(netcfg) print("---- ^^ modified ^^ ---- vv original vv ----") pprint.pprint(expected_cfg) - self.assertEqual(expected_cfg, netcfg) + assert expected_cfg == netcfg def _mock_v2_urls(httpretty): @@ -557,7 +690,6 @@ def _mock_no_v2_urls(httpretty): class TestReadOpcMetadata: # See https://docs.pytest.org/en/stable/example # /parametrize.html#parametrizing-conditional-raising - does_not_raise = ExitStack @mock.patch("cloudinit.url_helper.time.sleep", lambda _: None) @pytest.mark.parametrize( @@ -636,7 +768,29 @@ def test_retries( with expectation: assert expected_body == oracle.read_opc_metadata().instance_data + # No need to actually wait between retries in the tests + @mock.patch("cloudinit.url_helper.time.sleep", lambda _: None) + def test_fetch_vnics_error(self, caplog): + def mocked_fetch(*args, path="instance", **kwargs): + if path == "vnics": + raise UrlError("cause") + + with mock.patch(DS_PATH + "._fetch", side_effect=mocked_fetch): + opc_metadata = oracle.read_opc_metadata(fetch_vnics_data=True) + assert None is opc_metadata.vnics_data + assert ( + logging.WARNING, + "Failed to fetch IMDS network configuration!", + ) == caplog.record_tuples[-2][1:] + +@pytest.mark.parametrize( + "", + [ + pytest.param(marks=pytest.mark.is_iscsi(True), id="iscsi"), + pytest.param(marks=pytest.mark.is_iscsi(False), id="non-iscsi"), + ], +) class TestCommon_GetDataBehaviour: """This test class tests behaviour common to iSCSI and non-iSCSI root. @@ -649,33 +803,14 @@ class is implicitly also testing all iSCSI root behaviour so there is no separate class for that case.) """ - @pytest.fixture(params=[True, False]) - def parameterized_oracle_ds(self, request, oracle_ds): - """oracle_ds parameterized for iSCSI and non-iSCSI root respectively""" - is_iscsi_root = request.param - with ExitStack() as stack: - stack.enter_context( - mock.patch( - DS_PATH + "._is_iscsi_root", return_value=is_iscsi_root - ) - ) - if not is_iscsi_root: - stack.enter_context( - mock.patch(DS_PATH + ".net.find_fallback_nic") - ) - stack.enter_context( - mock.patch(DS_PATH + ".dhcp.EphemeralDHCPv4") - ) - yield oracle_ds - @mock.patch( DS_PATH + "._is_platform_viable", mock.Mock(return_value=False) ) def test_false_if_platform_not_viable( self, - parameterized_oracle_ds, + oracle_ds, ): - assert not parameterized_oracle_ds._get_data() + assert not oracle_ds._get_data() @pytest.mark.parametrize( "keyname,expected_value", @@ -699,10 +834,10 @@ def test_metadata_keys_set_correctly( self, keyname, expected_value, - parameterized_oracle_ds, + oracle_ds, ): - assert parameterized_oracle_ds._get_data() - assert expected_value == parameterized_oracle_ds.metadata[keyname] + assert oracle_ds._get_data() + assert expected_value == oracle_ds.metadata[keyname] @pytest.mark.parametrize( "attribute_name,expected_value", @@ -722,12 +857,10 @@ def test_attributes_set_correctly( self, attribute_name, expected_value, - parameterized_oracle_ds, + oracle_ds, ): - assert parameterized_oracle_ds._get_data() - assert expected_value == getattr( - parameterized_oracle_ds, attribute_name - ) + assert oracle_ds._get_data() + assert expected_value == getattr(oracle_ds, attribute_name) @pytest.mark.parametrize( "ssh_keys,expected_value", @@ -746,7 +879,7 @@ def test_attributes_set_correctly( ], ) def test_public_keys_handled_correctly( - self, ssh_keys, expected_value, parameterized_oracle_ds + self, ssh_keys, expected_value, oracle_ds ): instance_data = json.loads(OPC_V1_METADATA) if ssh_keys is None: @@ -758,14 +891,10 @@ def test_public_keys_handled_correctly( DS_PATH + ".read_opc_metadata", mock.Mock(return_value=metadata), ): - assert parameterized_oracle_ds._get_data() - assert ( - expected_value == parameterized_oracle_ds.get_public_ssh_keys() - ) + assert oracle_ds._get_data() + assert expected_value == oracle_ds.get_public_ssh_keys() - def test_missing_user_data_handled_gracefully( - self, parameterized_oracle_ds - ): + def test_missing_user_data_handled_gracefully(self, oracle_ds): instance_data = json.loads(OPC_V1_METADATA) del instance_data["metadata"]["user_data"] metadata = OpcMetadata(None, instance_data, None) @@ -773,13 +902,11 @@ def test_missing_user_data_handled_gracefully( DS_PATH + ".read_opc_metadata", mock.Mock(return_value=metadata), ): - assert parameterized_oracle_ds._get_data() + assert oracle_ds._get_data() - assert parameterized_oracle_ds.userdata_raw is None + assert oracle_ds.userdata_raw is None - def test_missing_metadata_handled_gracefully( - self, parameterized_oracle_ds - ): + def test_missing_metadata_handled_gracefully(self, oracle_ds): instance_data = json.loads(OPC_V1_METADATA) del instance_data["metadata"] metadata = OpcMetadata(None, instance_data, None) @@ -787,17 +914,17 @@ def test_missing_metadata_handled_gracefully( DS_PATH + ".read_opc_metadata", mock.Mock(return_value=metadata), ): - assert parameterized_oracle_ds._get_data() + assert oracle_ds._get_data() - assert parameterized_oracle_ds.userdata_raw is None - assert [] == parameterized_oracle_ds.get_public_ssh_keys() + assert oracle_ds.userdata_raw is None + assert [] == oracle_ds.get_public_ssh_keys() -@mock.patch(DS_PATH + "._is_iscsi_root", lambda: False) +@pytest.mark.is_iscsi(False) class TestNonIscsiRoot_GetDataBehaviour: - @mock.patch(DS_PATH + ".dhcp.EphemeralDHCPv4") + @mock.patch(DS_PATH + ".ephemeral.EphemeralDHCPv4") @mock.patch(DS_PATH + ".net.find_fallback_nic") - def test_read_opc_metadata_called_with_ephemeral_dhcp( + def test_run_net_files( self, m_find_fallback_nic, m_EphemeralDHCPv4, oracle_ds ): in_context_manager = False @@ -837,74 +964,122 @@ def assert_in_context_manager(**kwargs): ) ] == m_EphemeralDHCPv4.call_args_list + @mock.patch(DS_PATH + ".ephemeral.EphemeralDHCPv4") + @mock.patch(DS_PATH + ".net.find_fallback_nic") + def test_read_opc_metadata_called_with_ephemeral_dhcp( + self, m_find_fallback_nic, m_EphemeralDHCPv4, oracle_ds + ): + in_context_manager = False -@mock.patch(DS_PATH + ".get_interfaces_by_mac", lambda: {}) -@mock.patch(DS_PATH + ".cmdline.read_initramfs_config") -class TestNetworkConfig: - def test_network_config_cached(self, m_read_initramfs_config, oracle_ds): - """.network_config should be cached""" - assert 0 == m_read_initramfs_config.call_count - oracle_ds.network_config # pylint: disable=pointless-statement - assert 1 == m_read_initramfs_config.call_count - oracle_ds.network_config # pylint: disable=pointless-statement - assert 1 == m_read_initramfs_config.call_count + def enter_context_manager(): + nonlocal in_context_manager + in_context_manager = True - def test_network_cmdline(self, m_read_initramfs_config, oracle_ds): - """network_config should prefer initramfs config over fallback""" - ncfg = {"version": 1, "config": [{"a": "b"}]} - m_read_initramfs_config.return_value = copy.deepcopy(ncfg) + def exit_context_manager(*args): + nonlocal in_context_manager + in_context_manager = False - assert ncfg == oracle_ds.network_config - assert 0 == oracle_ds.distro.generate_fallback_config.call_count + m_EphemeralDHCPv4.return_value.__enter__.side_effect = ( + enter_context_manager + ) + m_EphemeralDHCPv4.return_value.__exit__.side_effect = ( + exit_context_manager + ) - def test_network_fallback(self, m_read_initramfs_config, oracle_ds): - """network_config should prefer initramfs config over fallback""" - ncfg = {"version": 1, "config": [{"a": "b"}]} + def assert_in_context_manager(**kwargs): + assert in_context_manager + return mock.MagicMock() - m_read_initramfs_config.return_value = None - oracle_ds.distro.generate_fallback_config.return_value = copy.deepcopy( - ncfg - ) + with mock.patch( + DS_PATH + ".read_opc_metadata", + mock.Mock(side_effect=assert_in_context_manager), + ): + assert oracle_ds._get_data() + + assert [ + mock.call( + iface=m_find_fallback_nic.return_value, + connectivity_url_data={ + "headers": {"Authorization": "Bearer Oracle"}, + "url": "http://169.254.169.254/opc/v2/instance/", + }, + ) + ] == m_EphemeralDHCPv4.call_args_list - assert ncfg == oracle_ds.network_config + +@mock.patch(DS_PATH + ".get_interfaces_by_mac", return_value={}) +class TestNetworkConfig: + def test_network_config_cached(self, m_get_interfaces_by_mac, oracle_ds): + """.network_config should be cached""" + assert 0 == oracle_ds._get_iscsi_config.call_count + oracle_ds.network_config # pylint: disable=pointless-statement + assert 1 == oracle_ds._get_iscsi_config.call_count + oracle_ds.network_config # pylint: disable=pointless-statement + assert 1 == oracle_ds._get_iscsi_config.call_count @pytest.mark.parametrize( - "configure_secondary_nics,expect_secondary_nics", - [(True, True), (False, False), (None, False)], + "configure_secondary_nics,is_iscsi,expected_set_primary", + [ + pytest.param( + True, + True, + [mock.call(False)], + marks=pytest.mark.is_iscsi(True), + ), + pytest.param( + True, + False, + [mock.call(True)], + marks=pytest.mark.is_iscsi(False), + ), + pytest.param(False, True, [], marks=pytest.mark.is_iscsi(True)), + pytest.param( + False, + False, + [mock.call(True)], + marks=pytest.mark.is_iscsi(False), + ), + pytest.param(None, True, [], marks=pytest.mark.is_iscsi(True)), + pytest.param( + None, + False, + [mock.call(True)], + marks=pytest.mark.is_iscsi(False), + ), + ], ) def test_secondary_nic_addition( self, - m_read_initramfs_config, + m_get_interfaces_by_mac, configure_secondary_nics, - expect_secondary_nics, + is_iscsi, + expected_set_primary, oracle_ds, ): """Test that _add_network_config_from_opc_imds is called as expected (configure_secondary_nics=None is used to test the default behaviour.) """ - m_read_initramfs_config.return_value = {"version": 1, "config": []} if configure_secondary_nics is not None: oracle_ds.ds_cfg[ "configure_secondary_nics" ] = configure_secondary_nics - def side_effect(self): - self._network_config["secondary_added"] = mock.sentinel.needle - oracle_ds._vnics_data = "DummyData" with mock.patch.object( - oracle.DataSourceOracle, + oracle_ds, "_add_network_config_from_opc_imds", - new=side_effect, - ): - was_secondary_added = "secondary_added" in oracle_ds.network_config - assert expect_secondary_nics == was_secondary_added + ) as m_add_network_config_from_opc_imds: + oracle_ds.network_config # pylint: disable=pointless-statement + assert ( + expected_set_primary + == m_add_network_config_from_opc_imds.call_args_list + ) def test_secondary_nic_failure_isnt_blocking( self, - m_read_initramfs_config, + m_get_interfaces_by_mac, caplog, oracle_ds, ): @@ -917,15 +1092,88 @@ def test_secondary_nic_failure_isnt_blocking( side_effect=Exception(), ): network_config = oracle_ds.network_config - assert network_config == m_read_initramfs_config.return_value - assert "Failed to parse secondary network configuration" in caplog.text + assert network_config == oracle_ds._get_iscsi_config.return_value + assert 2 == caplog.text.count( + "Failed to parse IMDS network configuration" + ) - def test_ds_network_cfg_preferred_over_initramfs(self, _m): + def test_ds_network_cfg_preferred_over_initramfs( + self, m_get_interfaces_by_mac + ): """Ensure that DS net config is preferred over initramfs config""" config_sources = oracle.DataSourceOracle.network_config_sources ds_idx = config_sources.index(NetworkConfigSource.DS) initramfs_idx = config_sources.index(NetworkConfigSource.INITRAMFS) assert ds_idx < initramfs_idx + @pytest.mark.parametrize("set_primary", [True, False]) + def test__add_network_config_from_opc_imds_no_vnics_data( + self, + m_get_interfaces_by_mac, + set_primary, + oracle_ds, + caplog, + ): + assert not oracle_ds._has_network_config() + with mock.patch.object(oracle_ds, "_vnics_data", None): + oracle_ds._add_network_config_from_opc_imds(set_primary) + assert not oracle_ds._has_network_config() + assert ( + logging.WARNING, + "NIC data is UNSET but should not be", + ) == caplog.record_tuples[-1][1:] + + def test_missing_mac_skipped( + self, + m_get_interfaces_by_mac, + oracle_ds, + caplog, + ): + """If no intefaces by mac found, then _network_config not setted and + correct logs. + """ + vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) + assert not oracle_ds._has_network_config() + with mock.patch.object(oracle_ds, "_vnics_data", vnics_data): + oracle_ds._add_network_config_from_opc_imds(set_primary=True) + assert not oracle_ds._has_network_config() + assert ( + logging.WARNING, + "Interface with MAC 02:00:17:05:d1:db not found; skipping", + ) == caplog.record_tuples[-2][1:] + assert ( + logging.WARNING, + f"Interface with MAC {MAC_ADDR} not found; skipping", + ) == caplog.record_tuples[-1][1:] + + @pytest.mark.parametrize("set_primary", [True, False]) + def test_nics( + self, + m_get_interfaces_by_mac, + set_primary, + oracle_ds, + caplog, + mocker, + ): + """Correct number of configs added""" + vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) + if set_primary: + assert not oracle_ds._has_network_config() + else: + # Simulate primary config was taken from iscsi + oracle_ds._network_config = copy.deepcopy(KLIBC_NET_CFG) + + mocker.patch( + DS_PATH + ".get_interfaces_by_mac", + return_value={"02:00:17:05:d1:db": "eth_0", MAC_ADDR: "name_1"}, + ) + mocker.patch.object(oracle_ds, "_vnics_data", vnics_data) + + oracle_ds._add_network_config_from_opc_imds(set_primary) + assert 2 == len( + oracle_ds._network_config["config"] + ), "Config not added" + assert "" == caplog.text + # vi: ts=4 expandtab diff --git a/tests/unittests/sources/test_ovf.py b/tests/unittests/sources/test_ovf.py index 68e558924..d949071d2 100644 --- a/tests/unittests/sources/test_ovf.py +++ b/tests/unittests/sources/test_ovf.py @@ -13,9 +13,6 @@ from cloudinit.helpers import Paths from cloudinit.safeyaml import YAMLError from cloudinit.sources import DataSourceOVF as dsovf -from cloudinit.sources.helpers.vmware.imc.config_custom_script import ( - CustomScriptNotFound, -) from tests.unittests.helpers import CiTestCase, mock, wrap_and_call MPATH = "cloudinit.sources.DataSourceOVF." diff --git a/tests/unittests/sources/test_scaleway.py b/tests/unittests/sources/test_scaleway.py index 52bcbc177..64c785d66 100644 --- a/tests/unittests/sources/test_scaleway.py +++ b/tests/unittests/sources/test_scaleway.py @@ -236,7 +236,7 @@ def test_metadata_ok(self, sleep, m_get_cmdline, dhcpv4): ].sort(), ) self.assertEqual( - self.datasource.get_hostname(), + self.datasource.get_hostname().hostname, MetadataResponses.FAKE_METADATA["hostname"], ) self.assertEqual( diff --git a/tests/unittests/sources/test_smartos.py b/tests/unittests/sources/test_smartos.py index 55239c4ef..702a67f7e 100644 --- a/tests/unittests/sources/test_smartos.py +++ b/tests/unittests/sources/test_smartos.py @@ -23,8 +23,9 @@ import uuid from binascii import crc32 +import serial + from cloudinit import helpers as c_helpers -from cloudinit import serial from cloudinit.event import EventScope, EventType from cloudinit.sources import DataSourceSmartOS from cloudinit.sources.DataSourceSmartOS import SERIAL_DEVICE, SMARTOS_ENV_KVM @@ -44,14 +45,6 @@ skipIf, ) -try: - import serial as _pyserial - - assert _pyserial # avoid pyflakes error F401: import unused - HAS_PYSERIAL = True -except ImportError: - HAS_PYSERIAL = False - DSMOS = "cloudinit.sources.DataSourceSmartOS" SDC_NICS = json.loads( """ @@ -1357,7 +1350,6 @@ def test_routes_on_all_nics(self): os.access(SERIAL_DEVICE, os.W_OK), "Requires write access to " + SERIAL_DEVICE, ) -@unittest.skipUnless(HAS_PYSERIAL is True, "pyserial not available") class TestSerialConcurrency(CiTestCase): """ This class tests locking on an actual serial port, and as such can only diff --git a/tests/unittests/sources/test_upcloud.py b/tests/unittests/sources/test_upcloud.py index e1125b652..317cb6386 100644 --- a/tests/unittests/sources/test_upcloud.py +++ b/tests/unittests/sources/test_upcloud.py @@ -216,8 +216,8 @@ def get_ds(self, get_sysinfo=_mock_dmi): @mock.patch("cloudinit.sources.helpers.upcloud.read_metadata") @mock.patch("cloudinit.net.find_fallback_nic") - @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") - @mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") def test_network_configured_metadata( self, m_net, m_dhcp, m_fallback_nic, mock_readmd ): diff --git a/tests/unittests/sources/test_vmware.py b/tests/unittests/sources/test_vmware.py index 3579041ab..b3663b0a9 100644 --- a/tests/unittests/sources/test_vmware.py +++ b/tests/unittests/sources/test_vmware.py @@ -87,6 +87,8 @@ class TestDataSourceVMware(CiTestCase): Test common functionality that is not transport specific. """ + with_logs = True + def setUp(self): super(TestDataSourceVMware, self).setUp() self.tmp = self.tmp_dir() @@ -141,6 +143,78 @@ def test_get_host_info_dual(self, m_fn_ipaddr): host_info[DataSourceVMware.LOCAL_IPV6] == "2001:db8::::::8888" ) + @mock.patch("cloudinit.sources.DataSourceVMware.get_host_info") + def test_wait_on_network(self, m_fn): + metadata = { + DataSourceVMware.WAIT_ON_NETWORK: { + DataSourceVMware.WAIT_ON_NETWORK_IPV4: True, + DataSourceVMware.WAIT_ON_NETWORK_IPV6: False, + }, + } + m_fn.side_effect = [ + { + "hostname": "host.cloudinit.test", + "local-hostname": "host.cloudinit.test", + "local_hostname": "host.cloudinit.test", + "network": { + "interfaces": { + "by-ipv4": {}, + "by-ipv6": {}, + "by-mac": { + "aa:bb:cc:dd:ee:ff": {"ipv4": [], "ipv6": []} + }, + }, + }, + }, + { + "hostname": "host.cloudinit.test", + "local-hostname": "host.cloudinit.test", + "local-ipv4": "10.10.10.1", + "local_hostname": "host.cloudinit.test", + "network": { + "interfaces": { + "by-ipv4": { + "10.10.10.1": { + "mac": "aa:bb:cc:dd:ee:ff", + "netmask": "255.255.255.0", + } + }, + "by-mac": { + "aa:bb:cc:dd:ee:ff": { + "ipv4": [ + { + "addr": "10.10.10.1", + "broadcast": "10.10.10.255", + "netmask": "255.255.255.0", + } + ], + "ipv6": [], + } + }, + }, + }, + }, + ] + + host_info = DataSourceVMware.wait_on_network(metadata) + + logs = self.logs.getvalue() + expected_logs = [ + "DEBUG: waiting on network: wait4=True, " + + "ready4=False, wait6=False, ready6=False\n", + "DEBUG: waiting on network complete\n", + ] + for log in expected_logs: + self.assertIn(log, logs) + + self.assertTrue(host_info) + self.assertTrue(host_info["hostname"]) + self.assertTrue(host_info["hostname"] == "host.cloudinit.test") + self.assertTrue(host_info["local-hostname"]) + self.assertTrue(host_info["local_hostname"]) + self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4]) + self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1") + class TestDataSourceVMwareEnvVars(FilesystemMockingTestCase): """ @@ -418,7 +492,9 @@ def test_ds_invalid_on_non_vmware_platform(self, m_fn): def assert_metadata(test_obj, ds, metadata): test_obj.assertEqual(metadata.get("instance-id"), ds.get_instance_id()) - test_obj.assertEqual(metadata.get("local-hostname"), ds.get_hostname()) + test_obj.assertEqual( + metadata.get("local-hostname"), ds.get_hostname().hostname + ) expected_public_keys = metadata.get("public_keys") if not isinstance(expected_public_keys, list): diff --git a/tests/unittests/sources/test_vultr.py b/tests/unittests/sources/test_vultr.py index c8398579d..5f2ccd4a9 100644 --- a/tests/unittests/sources/test_vultr.py +++ b/tests/unittests/sources/test_vultr.py @@ -344,9 +344,15 @@ def override_exit(self, excp_type, excp_value, excp_traceback): return # Test interface seeking to ensure we are able to find the correct one - @mock.patch("cloudinit.net.dhcp.EphemeralDHCPv4.__init__", ephemeral_init) - @mock.patch("cloudinit.net.dhcp.EphemeralDHCPv4.__enter__", override_enter) - @mock.patch("cloudinit.net.dhcp.EphemeralDHCPv4.__exit__", override_exit) + @mock.patch( + "cloudinit.net.ephemeral.EphemeralDHCPv4.__init__", ephemeral_init + ) + @mock.patch( + "cloudinit.net.ephemeral.EphemeralDHCPv4.__enter__", override_enter + ) + @mock.patch( + "cloudinit.net.ephemeral.EphemeralDHCPv4.__exit__", override_exit + ) @mock.patch("cloudinit.sources.helpers.vultr.check_route") @mock.patch("cloudinit.sources.helpers.vultr.is_vultr") @mock.patch("cloudinit.sources.helpers.vultr.read_metadata") @@ -377,10 +383,15 @@ def test_interface_seek( # Test route checking sucessful DHCPs @mock.patch("cloudinit.sources.helpers.vultr.check_route", check_route) @mock.patch( - "cloudinit.net.dhcp.EphemeralDHCPv4.__init__", ephemeral_init_always + "cloudinit.net.ephemeral.EphemeralDHCPv4.__init__", + ephemeral_init_always, + ) + @mock.patch( + "cloudinit.net.ephemeral.EphemeralDHCPv4.__enter__", override_enter + ) + @mock.patch( + "cloudinit.net.ephemeral.EphemeralDHCPv4.__exit__", override_exit ) - @mock.patch("cloudinit.net.dhcp.EphemeralDHCPv4.__enter__", override_enter) - @mock.patch("cloudinit.net.dhcp.EphemeralDHCPv4.__exit__", override_exit) @mock.patch("cloudinit.sources.helpers.vultr.get_interface_list") @mock.patch("cloudinit.sources.helpers.vultr.is_vultr") @mock.patch("cloudinit.sources.helpers.vultr.read_metadata") diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 0ed8a1201..44a06b2cd 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -5,9 +5,11 @@ import shutil import tempfile +import pytest + from cloudinit import handlers, helpers, settings, url_helper, util from cloudinit.cmd import main -from tests.unittests.helpers import CiTestCase, ExitStack, TestCase, mock +from tests.unittests.helpers import ExitStack, TestCase, mock class FakeModule(handlers.Handler): @@ -218,65 +220,117 @@ def test_exception_is_caught(self): ) -class TestCmdlineUrl(CiTestCase): +class FakeResponse: + def __init__(self, content, status_code=200): + self._content = content + self._remaining_content = content + self.status_code = status_code + self.encoding = None + + @property + def content(self): + return self._remaining_content + + def iter_content(self, chunk_size, *_, **__): + iterators = [iter(self._content)] * chunk_size + for chunk in zip(*iterators): + self._remaining_content = self._remaining_content[chunk_size:] + yield bytes(chunk) + + +class TestCmdlineUrl: def test_parse_cmdline_url_nokey_raises_keyerror(self): - self.assertRaises( - KeyError, main.parse_cmdline_url, "root=foo bar single" - ) + with pytest.raises(KeyError): + main.parse_cmdline_url("root=foo bar single") def test_parse_cmdline_url_found(self): cmdline = "root=foo bar single url=http://example.com arg1 -v" - self.assertEqual( - ("url", "http://example.com"), main.parse_cmdline_url(cmdline) - ) + assert ("url", "http://example.com") == main.parse_cmdline_url(cmdline) @mock.patch("cloudinit.cmd.main.url_helper.read_file_or_url") - def test_invalid_content(self, m_read): + def test_invalid_content(self, m_read, tmpdir): key = "cloud-config-url" url = "http://example.com/foo" cmdline = "ro %s=%s bar=1" % (key, url) m_read.return_value = url_helper.StringResponse(b"unexpected blob") - fpath = self.tmp_path("ccfile") + fpath = tmpdir.join("ccfile") lvl, msg = main.attempt_cmdline_url( fpath, network=True, cmdline=cmdline ) - self.assertEqual(logging.WARN, lvl) - self.assertIn(url, msg) - self.assertFalse(os.path.exists(fpath)) + assert logging.WARN == lvl + assert url in msg + assert False is os.path.exists(fpath) @mock.patch("cloudinit.cmd.main.url_helper.read_file_or_url") - def test_valid_content(self, m_read): + def test_invalid_content_url(self, m_read, tmpdir): + key = "cloud-config-url" + url = "http://example.com/foo" + cmdline = "ro %s=%s bar=1" % (key, url) + response = mock.Mock() + response.iter_content.return_value = iter( + (b"unexpected blob", StopIteration) + ) + response.status_code = 200 + m_read.return_value = url_helper.UrlResponse(response) + + fpath = tmpdir.join("ccfile") + lvl, msg = main.attempt_cmdline_url( + fpath, network=True, cmdline=cmdline + ) + assert logging.WARN == lvl + assert url in msg + assert False is os.path.exists(fpath) + + @mock.patch("cloudinit.cmd.main.url_helper.read_file_or_url") + def test_valid_content(self, m_read, tmpdir): url = "http://example.com/foo" payload = b"#cloud-config\nmydata: foo\nbar: wark\n" cmdline = "ro %s=%s bar=1" % ("cloud-config-url", url) m_read.return_value = url_helper.StringResponse(payload) - fpath = self.tmp_path("ccfile") + fpath = tmpdir.join("ccfile") + lvl, msg = main.attempt_cmdline_url( + fpath, network=True, cmdline=cmdline + ) + assert util.load_file(fpath, decode=False) == payload + assert logging.INFO == lvl + assert url in msg + + @mock.patch("cloudinit.cmd.main.url_helper.read_file_or_url") + def test_valid_content_url(self, m_read, tmpdir): + url = "http://example.com/foo" + payload = b"#cloud-config\nmydata: foo\nbar: wark\n" + cmdline = "ro %s=%s bar=1" % ("cloud-config-url", url) + + response = FakeResponse(payload) + m_read.return_value = url_helper.UrlResponse(response) + + fpath = tmpdir.join("ccfile") lvl, msg = main.attempt_cmdline_url( fpath, network=True, cmdline=cmdline ) - self.assertEqual(util.load_file(fpath, decode=False), payload) - self.assertEqual(logging.INFO, lvl) - self.assertIn(url, msg) + assert util.load_file(fpath, decode=False) == payload + assert logging.INFO == lvl + assert url in msg @mock.patch("cloudinit.cmd.main.url_helper.read_file_or_url") - def test_no_key_found(self, m_read): + def test_no_key_found(self, m_read, tmpdir): cmdline = "ro mykey=http://example.com/foo root=foo" - fpath = self.tmp_path("ccpath") + fpath = tmpdir.join("ccfile") lvl, _msg = main.attempt_cmdline_url( fpath, network=True, cmdline=cmdline ) m_read.assert_not_called() - self.assertFalse(os.path.exists(fpath)) - self.assertEqual(logging.DEBUG, lvl) + assert False is os.path.exists(fpath) + assert logging.DEBUG == lvl @mock.patch("cloudinit.cmd.main.url_helper.read_file_or_url") - def test_exception_warns(self, m_read): + def test_exception_warns(self, m_read, tmpdir): url = "http://example.com/foo" cmdline = "ro cloud-config-url=%s root=LABEL=bar" % url - fpath = self.tmp_path("ccfile") + fpath = tmpdir.join("ccfile") m_read.side_effect = url_helper.UrlError( cause="Unexpected Error", url="http://example.com/foo" ) @@ -284,9 +338,9 @@ def test_exception_warns(self, m_read): lvl, msg = main.attempt_cmdline_url( fpath, network=True, cmdline=cmdline ) - self.assertEqual(logging.WARN, lvl) - self.assertIn(url, msg) - self.assertFalse(os.path.exists(fpath)) + assert logging.WARN == lvl + assert url in msg + assert False is os.path.exists(fpath) # vi: ts=4 expandtab diff --git a/tests/unittests/test_apport.py b/tests/unittests/test_apport.py new file mode 100644 index 000000000..a2c866b95 --- /dev/null +++ b/tests/unittests/test_apport.py @@ -0,0 +1,23 @@ +from tests.unittests.helpers import mock + +M_PATH = "cloudinit.apport." + + +class TestApport: + def test_attach_user_data(self, mocker, tmpdir): + m_hookutils = mock.Mock() + mocker.patch.dict("sys.modules", {"apport.hookutils": m_hookutils}) + user_data_file = tmpdir.join("instance", "user-data.txt") + mocker.patch( + M_PATH + "_get_user_data_file", return_value=user_data_file + ) + + from cloudinit import apport + + ui = mock.Mock() + ui.yesno.return_value = True + report = object() + apport.attach_user_data(report, ui) + assert [ + mock.call(report, user_data_file, "user_data.txt") + ] == m_hookutils.attach_file.call_args_list diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 7846d0d33..04f5f4573 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -5,22 +5,27 @@ import os from collections import namedtuple +import pytest + +from cloudinit import helpers from cloudinit.cmd import main as cli from cloudinit.util import load_file, load_json from tests.unittests import helpers as test_helpers mock = test_helpers.mock +M_PATH = "cloudinit.cmd.main." -class TestCLI(test_helpers.FilesystemMockingTestCase): - with_logs = True +@pytest.fixture(autouse=False) +def mock_get_user_data_file(mocker, tmpdir): + yield mocker.patch( + "cloudinit.cmd.devel.logs._get_user_data_file", + return_value=tmpdir.join("cloud"), + ) - def setUp(self): - super(TestCLI, self).setUp() - self.stderr = io.StringIO() - self.patchStdoutAndStderr(stderr=self.stderr) +class TestCLI: def _call_main(self, sysv_args=None): if not sysv_args: sysv_args = ["cloud-init"] @@ -29,57 +34,48 @@ def _call_main(self, sysv_args=None): except SystemExit as e: return e.code - def test_status_wrapper_errors_on_invalid_name(self): - """status_wrapper will error when the name parameter is not valid. - - Valid name values are only init and modules. - """ - tmpd = self.tmp_dir() - data_d = self.tmp_path("data", tmpd) - link_d = self.tmp_path("link", tmpd) + @pytest.mark.parametrize( + "action,name,match", + [ + pytest.param( + "doesnotmatter", + "init1", + "^unknown name: init1$", + id="invalid_name", + ), + pytest.param( + "modules_name", + "modules", + "^Invalid cloud init mode specified 'modules-bogusmode'$", + id="invalid_modes", + ), + ], + ) + def test_status_wrapper_errors(self, action, name, match, caplog, tmpdir): + data_d = tmpdir.join("data") + link_d = tmpdir.join("link") FakeArgs = namedtuple("FakeArgs", ["action", "local", "mode"]) def myaction(): raise Exception("Should not call myaction") - myargs = FakeArgs(("doesnotmatter", myaction), False, "bogusmode") - with self.assertRaises(ValueError) as cm: - cli.status_wrapper("init1", myargs, data_d, link_d) - self.assertEqual("unknown name: init1", str(cm.exception)) - self.assertNotIn("Should not call myaction", self.logs.getvalue()) - - def test_status_wrapper_errors_on_invalid_modes(self): - """status_wrapper will error if a parameter combination is invalid.""" - tmpd = self.tmp_dir() - data_d = self.tmp_path("data", tmpd) - link_d = self.tmp_path("link", tmpd) - FakeArgs = namedtuple("FakeArgs", ["action", "local", "mode"]) - - def myaction(): - raise Exception("Should not call myaction") + myargs = FakeArgs((action, myaction), False, "bogusmode") + with pytest.raises(ValueError, match=match): + cli.status_wrapper(name, myargs, data_d, link_d) + assert "Should not call myaction" not in caplog.text - myargs = FakeArgs(("modules_name", myaction), False, "bogusmode") - with self.assertRaises(ValueError) as cm: - cli.status_wrapper("modules", myargs, data_d, link_d) - self.assertEqual( - "Invalid cloud init mode specified 'modules-bogusmode'", - str(cm.exception), - ) - self.assertNotIn("Should not call myaction", self.logs.getvalue()) - - def test_status_wrapper_init_local_writes_fresh_status_info(self): + def test_status_wrapper_init_local_writes_fresh_status_info(self, tmpdir): """When running in init-local mode, status_wrapper writes status.json. Old status and results artifacts are also removed. """ - tmpd = self.tmp_dir() - data_d = self.tmp_path("data", tmpd) - link_d = self.tmp_path("link", tmpd) - status_link = self.tmp_path("status.json", link_d) + data_d = tmpdir.join("data") + link_d = tmpdir.join("link") + status_link = link_d.join("status.json") # Write old artifacts which will be removed or updated. for _dir in data_d, link_d: test_helpers.populate_dir( - _dir, {"status.json": "old", "result.json": "old"} + str(_dir), {"status.json": "old", "result.json": "old"} ) FakeArgs = namedtuple("FakeArgs", ["action", "local", "mode"]) @@ -92,39 +88,63 @@ def myaction(name, args): cli.status_wrapper("init", myargs, data_d, link_d) # No errors reported in status status_v1 = load_json(load_file(status_link))["v1"] - self.assertEqual(["an error"], status_v1["init-local"]["errors"]) - self.assertEqual("SomeDatasource", status_v1["datasource"]) - self.assertFalse( - os.path.exists(self.tmp_path("result.json", data_d)), - "unexpected result.json found", - ) - self.assertFalse( - os.path.exists(self.tmp_path("result.json", link_d)), - "unexpected result.json link found", - ) + assert ["an error"] == status_v1["init-local"]["errors"] + assert "SomeDatasource" == status_v1["datasource"] + assert False is os.path.exists( + data_d.join("result.json") + ), "unexpected result.json found" + assert False is os.path.exists( + link_d.join("result.json") + ), "unexpected result.json link found" + + def test_status_wrapper_init_local_honor_cloud_dir(self, mocker, tmpdir): + """When running in init-local mode, status_wrapper honors cloud_dir.""" + cloud_dir = tmpdir.join("cloud") + paths = helpers.Paths({"cloud_dir": str(cloud_dir)}) + mocker.patch(M_PATH + "read_cfg_paths", return_value=paths) + data_d = cloud_dir.join("data") + link_d = tmpdir.join("link") + + FakeArgs = namedtuple("FakeArgs", ["action", "local", "mode"]) + + def myaction(name, args): + # Return an error to watch status capture them + return "SomeDatasource", ["an_error"] - def test_no_arguments_shows_usage(self): + myargs = FakeArgs(("ignored_name", myaction), True, "bogusmode") + cli.status_wrapper("init", myargs, link_d=link_d) # No explicit data_d + # Access cloud_dir directly + status_v1 = load_json(load_file(data_d.join("status.json")))["v1"] + assert ["an_error"] == status_v1["init-local"]["errors"] + assert "SomeDatasource" == status_v1["datasource"] + assert False is os.path.exists( + data_d.join("result.json") + ), "unexpected result.json found" + assert False is os.path.exists( + link_d.join("result.json") + ), "unexpected result.json link found" + + def test_no_arguments_shows_usage(self, capsys): exit_code = self._call_main() - self.assertIn("usage: cloud-init", self.stderr.getvalue()) - self.assertEqual(2, exit_code) + _out, err = capsys.readouterr() + assert "usage: cloud-init" in err + assert 2 == exit_code - def test_no_arguments_shows_error_message(self): + def test_no_arguments_shows_error_message(self, capsys): exit_code = self._call_main() - missing_subcommand_message = [ - "too few arguments", # python2.7 msg - "the following arguments are required: subcommand", # python3 msg - ] - error = self.stderr.getvalue() - matches = [msg in error for msg in missing_subcommand_message] - self.assertTrue( - any(matches), "Did not find error message for missing subcommand" + missing_subcommand_message = ( + "the following arguments are required: subcommand" ) - self.assertEqual(2, exit_code) + _out, err = capsys.readouterr() + assert ( + missing_subcommand_message in err + ), "Did not find error message for missing subcommand" + assert 2 == exit_code - def test_all_subcommands_represented_in_help(self): + def test_all_subcommands_represented_in_help(self, capsys): """All known subparsers are represented in the cloud-int help doc.""" self._call_main() - error = self.stderr.getvalue() + _out, err = capsys.readouterr() expected_subcommands = [ "analyze", "clean", @@ -137,241 +157,188 @@ def test_all_subcommands_represented_in_help(self): "schema", ] for subcommand in expected_subcommands: - self.assertIn(subcommand, error) - - @mock.patch("cloudinit.cmd.main.status_wrapper") - def test_init_subcommand_parser(self, m_status_wrapper): - """The subcommand 'init' calls status_wrapper passing init.""" - self._call_main(["cloud-init", "init"]) - (name, parseargs) = m_status_wrapper.call_args_list[0][0] - self.assertEqual("init", name) - self.assertEqual("init", parseargs.subcommand) - self.assertEqual("init", parseargs.action[0]) - self.assertEqual("main_init", parseargs.action[1].__name__) + assert subcommand in err + @pytest.mark.parametrize("subcommand", ["init", "modules"]) @mock.patch("cloudinit.cmd.main.status_wrapper") - def test_modules_subcommand_parser(self, m_status_wrapper): - """The subcommand 'modules' calls status_wrapper passing modules.""" - self._call_main(["cloud-init", "modules"]) + def test_modules_subcommand_parser(self, m_status_wrapper, subcommand): + """The subcommand 'subcommand' calls status_wrapper passing modules.""" + self._call_main(["cloud-init", subcommand]) (name, parseargs) = m_status_wrapper.call_args_list[0][0] - self.assertEqual("modules", name) - self.assertEqual("modules", parseargs.subcommand) - self.assertEqual("modules", parseargs.action[0]) - self.assertEqual("main_modules", parseargs.action[1].__name__) - - def test_conditional_subcommands_from_entry_point_sys_argv(self): - """Subcommands from entry-point are properly parsed from sys.argv.""" - stdout = io.StringIO() - self.patchStdoutAndStderr(stdout=stdout) - - expected_errors = [ - "usage: cloud-init analyze", - "usage: cloud-init clean", - "usage: cloud-init collect-logs", - "usage: cloud-init devel", - "usage: cloud-init status", - "usage: cloud-init schema", - ] - conditional_subcommands = [ + assert subcommand == name + assert subcommand == parseargs.subcommand + assert subcommand == parseargs.action[0] + assert f"main_{subcommand}" == parseargs.action[1].__name__ + + @pytest.mark.parametrize( + "subcommand", + [ "analyze", "clean", "collect-logs", "devel", "status", "schema", - ] + ], + ) + def test_conditional_subcommands_from_entry_point_sys_argv( + self, subcommand, capsys, mock_get_user_data_file, tmpdir + ): + """Subcommands from entry-point are properly parsed from sys.argv.""" + expected_error = f"usage: cloud-init {subcommand}" # The cloud-init entrypoint calls main without passing sys_argv - for subcommand in conditional_subcommands: - with mock.patch("sys.argv", ["cloud-init", subcommand, "-h"]): - try: - cli.main() - except SystemExit as e: - self.assertEqual(0, e.code) # exit 2 on proper -h usage - for error_message in expected_errors: - self.assertIn(error_message, stdout.getvalue()) - - def test_analyze_subcommand_parser(self): - """The subcommand cloud-init analyze calls the correct subparser.""" - self._call_main(["cloud-init", "analyze"]) - # These subcommands only valid for cloud-init analyze script - expected_subcommands = ["blame", "show", "dump"] - error = self.stderr.getvalue() - for subcommand in expected_subcommands: - self.assertIn(subcommand, error) - - def test_collect_logs_subcommand_parser(self): - """The subcommand cloud-init collect-logs calls the subparser.""" - # Provide -h param to collect-logs to avoid having to mock behavior. - stdout = io.StringIO() - self.patchStdoutAndStderr(stdout=stdout) - self._call_main(["cloud-init", "collect-logs", "-h"]) - self.assertIn("usage: cloud-init collect-log", stdout.getvalue()) - - def test_clean_subcommand_parser(self): - """The subcommand cloud-init clean calls the subparser.""" - # Provide -h param to clean to avoid having to mock behavior. - stdout = io.StringIO() - self.patchStdoutAndStderr(stdout=stdout) - self._call_main(["cloud-init", "clean", "-h"]) - self.assertIn("usage: cloud-init clean", stdout.getvalue()) - - def test_status_subcommand_parser(self): - """The subcommand cloud-init status calls the subparser.""" - # Provide -h param to clean to avoid having to mock behavior. - stdout = io.StringIO() - self.patchStdoutAndStderr(stdout=stdout) - self._call_main(["cloud-init", "status", "-h"]) - self.assertIn("usage: cloud-init status", stdout.getvalue()) - - def test_subcommand_parser(self): + with mock.patch("sys.argv", ["cloud-init", subcommand, "-h"]): + try: + cli.main() + except SystemExit as e: + assert 0 == e.code # exit 2 on proper -h usage + out, _err = capsys.readouterr() + assert expected_error in out + + @pytest.mark.parametrize( + "subcommand", + [ + "clean", + "collect-logs", + "status", + ], + ) + def test_subcommand_parser(self, subcommand, mock_get_user_data_file): + """cloud-init `subcommand` calls its subparser.""" + # Provide -h param to `subcommand` to avoid having to mock behavior. + out = io.StringIO() + with contextlib.redirect_stdout(out): + self._call_main(["cloud-init", subcommand, "-h"]) + assert f"usage: cloud-init {subcommand}" in out.getvalue() + + @pytest.mark.parametrize( + "args,expected_subcommands", + [ + ([], ["schema"]), + (["analyze"], ["blame", "show", "dump"]), + ], + ) + def test_subcommand_parser_multi_arg( + self, args, expected_subcommands, capsys + ): """The subcommand cloud-init schema calls the correct subparser.""" - self._call_main(["cloud-init"]) - # These subcommands only valid for cloud-init schema script - expected_subcommands = ["schema"] - error = self.stderr.getvalue() + self._call_main(["cloud-init"] + args) + _out, err = capsys.readouterr() for subcommand in expected_subcommands: - self.assertIn(subcommand, error) + assert subcommand in err - def test_wb_schema_subcommand_parser(self): + def test_wb_schema_subcommand_parser(self, capsys): """The subcommand cloud-init schema calls the correct subparser.""" exit_code = self._call_main(["cloud-init", "schema"]) - self.assertEqual(1, exit_code) + _out, err = capsys.readouterr() + assert 1 == exit_code # Known whitebox output from schema subcommand - self.assertEqual( + assert ( "Error:\n" - "Expected one of --config-file, --system or --docs arguments\n", - self.stderr.getvalue(), + "Expected one of --config-file, --system or --docs arguments\n" + == err ) - def test_wb_schema_subcommand_doc_all_spot_check(self): - """Validate that doc content has correct values from known examples. - - Ensure that schema doc is returned - """ - - # Note: patchStdoutAndStderr() is convenient for reducing boilerplate, - # but inspecting the code for debugging is not ideal - # contextlib.redirect_stdout() provides similar behavior as a context - # manager - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - self._call_main(["cloud-init", "schema", "--docs", "all"]) - expected_doc_sections = [ - "**Supported distros:** all", - "**Supported distros:** almalinux, alpine, centos, " - "cloudlinux, debian, eurolinux, fedora, miraclelinux, " - "openEuler, opensuse, photon, rhel, rocky, sles, ubuntu, " - "virtuozzo", - "**Config schema**:\n **resize_rootfs:** " - "(``true``/``false``/``noblock``)", - "**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n", - ] - stdout = stdout.getvalue() - for expected in expected_doc_sections: - self.assertIn(expected, stdout) - - def test_wb_schema_subcommand_single_spot_check(self): - """Validate that doc content has correct values from known example. - - Validate 'all' arg - """ - - # Note: patchStdoutAndStderr() is convenient for reducing boilerplate, - # but inspecting the code for debugging is not ideal - # contextlib.redirect_stdout() provides similar behavior as a context - # manager - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - self._call_main(["cloud-init", "schema", "--docs", "cc_runcmd"]) - expected_doc_sections = [ - "Runcmd\n------\n**Summary:** Run arbitrary commands" - ] - stdout = stdout.getvalue() - for expected in expected_doc_sections: - self.assertIn(expected, stdout) - - def test_wb_schema_subcommand_multiple_spot_check(self): - """Validate that doc content has correct values from known example. - - Validate single arg - """ - - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - self._call_main( + @pytest.mark.parametrize( + "args,expected_doc_sections,is_error", + [ + pytest.param( + ["all"], + [ + "**Supported distros:** all", + "**Supported distros:** almalinux, alpine, centos, " + "cloudlinux, debian, eurolinux, fedora, miraclelinux, " + "openEuler, openmandriva, opensuse, photon, rhel, rocky, " + "sles, ubuntu, virtuozzo", + "**Config schema**:\n **resize_rootfs:** " + "(``true``/``false``/``noblock``)", + "**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n", + ], + False, + id="all_spot_check", + ), + pytest.param( + ["cc_runcmd"], + ["Runcmd\n------\n**Summary:** Run arbitrary commands"], + False, + id="single_spot_check", + ), + pytest.param( [ - "cloud-init", - "schema", - "--docs", "cc_runcmd", "cc_resizefs", - ] - ) - expected_doc_sections = [ - "Runcmd\n------\n**Summary:** Run arbitrary commands", - "Resizefs\n--------\n**Summary:** Resize filesystem", - ] - stdout = stdout.getvalue() - for expected in expected_doc_sections: - self.assertIn(expected, stdout) - - def test_wb_schema_subcommand_bad_arg_fails(self): - """Validate that doc content has correct values from known example. - - Validate multiple args - """ + ], + [ + "Runcmd\n------\n**Summary:** Run arbitrary commands", + "Resizefs\n--------\n**Summary:** Resize filesystem", + ], + False, + id="multiple_spot_check", + ), + pytest.param( + ["garbage_value"], + ["Invalid --docs value"], + True, + id="bad_arg_fails", + ), + ], + ) + def test_wb_schema_subcommand(self, args, expected_doc_sections, is_error): + """Validate that doc content has correct values.""" # Note: patchStdoutAndStderr() is convenient for reducing boilerplate, # but inspecting the code for debugging is not ideal # contextlib.redirect_stdout() provides similar behavior as a context # manager - stderr = io.StringIO() - with contextlib.redirect_stderr(stderr): - self._call_main( - ["cloud-init", "schema", "--docs", "garbage_value"] - ) - expected_doc_sections = ["Invalid --docs value"] - stderr = stderr.getvalue() + out_or_err = io.StringIO() + redirecter = ( + contextlib.redirect_stderr + if is_error + else contextlib.redirect_stdout + ) + with redirecter(out_or_err): + self._call_main(["cloud-init", "schema", "--docs"] + args) + out_or_err = out_or_err.getvalue() for expected in expected_doc_sections: - self.assertIn(expected, stderr) + assert expected in out_or_err @mock.patch("cloudinit.cmd.main.main_single") def test_single_subcommand(self, m_main_single): """The subcommand 'single' calls main_single with valid args.""" self._call_main(["cloud-init", "single", "--name", "cc_ntp"]) (name, parseargs) = m_main_single.call_args_list[0][0] - self.assertEqual("single", name) - self.assertEqual("single", parseargs.subcommand) - self.assertEqual("single", parseargs.action[0]) - self.assertFalse(parseargs.debug) - self.assertFalse(parseargs.force) - self.assertIsNone(parseargs.frequency) - self.assertEqual("cc_ntp", parseargs.name) - self.assertFalse(parseargs.report) + assert "single" == name + assert "single" == parseargs.subcommand + assert "single" == parseargs.action[0] + assert False is parseargs.debug + assert False is parseargs.force + assert None is parseargs.frequency + assert "cc_ntp" == parseargs.name + assert False is parseargs.report @mock.patch("cloudinit.cmd.main.dhclient_hook.handle_args") def test_dhclient_hook_subcommand(self, m_handle_args): """The subcommand 'dhclient-hook' calls dhclient_hook with args.""" self._call_main(["cloud-init", "dhclient-hook", "up", "eth0"]) (name, parseargs) = m_handle_args.call_args_list[0][0] - self.assertEqual("dhclient-hook", name) - self.assertEqual("dhclient-hook", parseargs.subcommand) - self.assertEqual("dhclient-hook", parseargs.action[0]) - self.assertFalse(parseargs.debug) - self.assertFalse(parseargs.force) - self.assertEqual("up", parseargs.event) - self.assertEqual("eth0", parseargs.interface) + assert "dhclient-hook" == name + assert "dhclient-hook" == parseargs.subcommand + assert "dhclient-hook" == parseargs.action[0] + assert False is parseargs.debug + assert False is parseargs.force + assert "up" == parseargs.event + assert "eth0" == parseargs.interface @mock.patch("cloudinit.cmd.main.main_features") def test_features_hook_subcommand(self, m_features): """The subcommand 'features' calls main_features with args.""" self._call_main(["cloud-init", "features"]) (name, parseargs) = m_features.call_args_list[0][0] - self.assertEqual("features", name) - self.assertEqual("features", parseargs.subcommand) - self.assertEqual("features", parseargs.action[0]) - self.assertFalse(parseargs.debug) - self.assertFalse(parseargs.force) + assert "features" == name + assert "features" == parseargs.subcommand + assert "features" == parseargs.action[0] + assert False is parseargs.debug + assert False is parseargs.force # : ts=4 expandtab diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 75c304a8c..eda04093c 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -16,23 +16,13 @@ from cloudinit import handlers from cloudinit import helpers as c_helpers -from cloudinit import log, safeyaml, sources, stages +from cloudinit import log, safeyaml, stages from cloudinit import user_data as ud from cloudinit import util from cloudinit.config.modules import Modules from cloudinit.settings import PER_INSTANCE from tests.unittests import helpers - -INSTANCE_ID = "i-testing" - - -class FakeDataSource(sources.DataSource): - def __init__(self, userdata=None, vendordata=None, vendordata2=None): - sources.DataSource.__init__(self, {}, None, None) - self.metadata = {"instance-id": INSTANCE_ID} - self.userdata_raw = userdata - self.vendordata_raw = vendordata - self.vendordata2_raw = vendordata2 +from tests.unittests.util import FakeDataSource def count_messages(root): diff --git a/tests/unittests/test_dmi.py b/tests/unittests/test_dmi.py index 6c28724ac..91d424c1f 100644 --- a/tests/unittests/test_dmi.py +++ b/tests/unittests/test_dmi.py @@ -68,7 +68,9 @@ def patch_mapping(self, new_mapping): ) def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self): - self.patch_mapping({"mapped-key": dmi.kdmi("mapped-value", None)}) + self.patch_mapping( + {"mapped-key": dmi.KernelNames("mapped-value", None)} + ) expected_dmi_value = "sys-used-correctly" self._create_sysfs_file("mapped-value", expected_dmi_value) self._configure_dmidecode_return("mapped-key", "wrong-wrong-wrong") diff --git a/tests/unittests/test_features.py b/tests/unittests/test_features.py index 794a96540..94c7ae13b 100644 --- a/tests/unittests/test_features.py +++ b/tests/unittests/test_features.py @@ -44,11 +44,15 @@ def create_override(request): class TestFeatures: + """default pytest-xdist behavior may fail due to these tests""" + + @pytest.mark.serial def test_feature_without_override(self): from cloudinit.features import ERROR_ON_USER_DATA_FAILURE assert ERROR_ON_USER_DATA_FAILURE is True + @pytest.mark.serial @pytest.mark.parametrize( "create_override", [{"ERROR_ON_USER_DATA_FAILURE": False}], @@ -59,6 +63,7 @@ def test_feature_with_override(self, create_override): assert ERROR_ON_USER_DATA_FAILURE is False + @pytest.mark.serial @pytest.mark.parametrize( "create_override", [{"SPAM": True}], indirect=True ) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 0d5623b4e..25f47f798 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -443,7 +443,7 @@ macaddress: 68:05:ca:64:d3:6c mtu: 9000 parameters: - gratuitious-arp: 1 + gratuitous-arp: 1 bond1: interfaces: - ens4 @@ -1248,9 +1248,8 @@ may-fail=false [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -1278,6 +1277,7 @@ DHCP=no [Address] Address=192.168.14.2/24 + [Address] Address=2001:1::1/64 """ ).rstrip(" "), @@ -1383,7 +1383,6 @@ [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::1/64 """ @@ -1416,9 +1415,8 @@ [ethernet] [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy [ipv4] method=auto @@ -1517,9 +1515,8 @@ [ethernet] [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -1750,7 +1747,6 @@ [ipv6] method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -1862,7 +1858,6 @@ [ipv6] method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -2683,7 +2678,6 @@ [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::1/64 route1=::/0,2001:4800:78ff:1b::1 @@ -2736,9 +2730,8 @@ xmit_hash_policy=layer3+4 [ipv6] - method=dhcp + method=auto may-fail=false - addr-gen-mode=stable-privacy """ ), @@ -2987,7 +2980,7 @@ parameters: down-delay: 10 fail-over-mac-policy: active - gratuitious-arp: 5 + gratuitous-arp: 5 mii-monitor-interval: 100 mode: active-backup primary: bond0s0 @@ -3095,7 +3088,7 @@ parameters: down-delay: 10 fail-over-mac-policy: active - gratuitious-arp: 5 + gratuitous-arp: 5 mii-monitor-interval: 100 mode: active-backup primary: bond0s0 @@ -3128,7 +3121,7 @@ parameters: down-delay: 10 fail-over-mac-policy: active - gratuitious-arp: 5 + gratuitous-arp: 5 mii-monitor-interval: 100 mode: active-backup primary: bond0s0 @@ -3342,7 +3335,6 @@ [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::1/92 route1=2001:67c::/32,2001:67c:1562::1 route2=3001:67c::/32,3001:67c:15::1 @@ -3463,7 +3455,6 @@ [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::bbbb/96 route1=::/0,2001:1::1 @@ -3641,7 +3632,6 @@ [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::100/96 """ @@ -3666,7 +3656,6 @@ [ipv6] method=manual may-fail=false - addr-gen-mode=stable-privacy address1=2001:1::101/96 """ @@ -6766,7 +6755,7 @@ def test_render_output_supports_both_grat_arp_spelling(self): entry = { "yaml": NETPLAN_BOND_GRAT_ARP, "expected_netplan": NETPLAN_BOND_GRAT_ARP.replace( - "gratuitous", "gratuitious" + "gratuitious", "gratuitous" ), } network_config = yaml.load(entry["yaml"]).get("network") @@ -7516,7 +7505,7 @@ class TestGetInterfaces(CiTestCase): "tun0": None, }, } - data = {} + data: dict = {} def _se_get_devicelist(self): return list(self.data["devices"]) @@ -7690,7 +7679,7 @@ class TestGetInterfacesByMac(CiTestCase): "tun0": None, }, } - data = {} + data: dict = {} def _se_get_devicelist(self): return list(self.data["devices"]) @@ -7900,7 +7889,7 @@ class TestGetIBHwaddrsByInterface(CiTestCase): }, "ib_hwaddr": {"ib0": {True: _ib_addr_eth_format, False: _ib_addr}}, } - data = {} + data: dict = {} def _mock_setup(self): self.data = copy.deepcopy(self._data) diff --git a/tests/unittests/test_net_activators.py b/tests/unittests/test_net_activators.py index 9eec74c91..afd9056af 100644 --- a/tests/unittests/test_net_activators.py +++ b/tests/unittests/test_net_activators.py @@ -6,6 +6,7 @@ from cloudinit.net.activators import ( DEFAULT_PRIORITY, + NAME_TO_ACTIVATOR, IfUpDownActivator, NetplanActivator, NetworkdActivator, @@ -35,7 +36,7 @@ dhcp4: true """ -NETPLAN_CALL_LIST = [ +NETPLAN_CALL_LIST: list = [ ((["netplan", "apply"],), {}), ] @@ -79,23 +80,23 @@ def unavailable_mocks(): class TestSearchAndSelect: - def test_defaults(self, available_mocks): - resp = search_activator() - assert resp == DEFAULT_PRIORITY + def test_empty_list(self, available_mocks): + resp = search_activator(priority=DEFAULT_PRIORITY, target=None) + assert resp == [NAME_TO_ACTIVATOR[name] for name in DEFAULT_PRIORITY] activator = select_activator() - assert activator == DEFAULT_PRIORITY[0] + assert activator == NAME_TO_ACTIVATOR[DEFAULT_PRIORITY[0]] def test_priority(self, available_mocks): - new_order = [NetplanActivator, NetworkManagerActivator] - resp = search_activator(priority=new_order) - assert resp == new_order + new_order = ["netplan", "network-manager"] + resp = search_activator(priority=new_order, target=None) + assert resp == [NAME_TO_ACTIVATOR[name] for name in new_order] activator = select_activator(priority=new_order) - assert activator == new_order[0] + assert activator == NAME_TO_ACTIVATOR[new_order[0]] def test_target(self, available_mocks): - search_activator(target="/tmp") + search_activator(priority=DEFAULT_PRIORITY, target="/tmp") assert "/tmp" == available_mocks.m_which.call_args[1]["target"] select_activator(target="/tmp") @@ -106,20 +107,22 @@ def test_target(self, available_mocks): return_value=False, ) def test_first_not_available(self, m_available, available_mocks): - resp = search_activator() - assert resp == DEFAULT_PRIORITY[1:] + resp = search_activator(priority=DEFAULT_PRIORITY, target=None) + assert resp == [ + NAME_TO_ACTIVATOR[activator] for activator in DEFAULT_PRIORITY[1:] + ] resp = select_activator() - assert resp == DEFAULT_PRIORITY[1] + assert resp == NAME_TO_ACTIVATOR[DEFAULT_PRIORITY[1]] def test_priority_not_exist(self, available_mocks): with pytest.raises(ValueError): - search_activator(priority=["spam", "eggs"]) + search_activator(priority=["spam", "eggs"], target=None) with pytest.raises(ValueError): select_activator(priority=["spam", "eggs"]) def test_none_available(self, unavailable_mocks): - resp = search_activator() + resp = search_activator(priority=DEFAULT_PRIORITY, target=None) assert resp == [] with pytest.raises(NoActivatorException): @@ -156,12 +159,12 @@ def test_available(self, activator, available_calls, available_mocks): assert available_mocks.m_which.call_args_list == available_calls -IF_UP_DOWN_BRING_UP_CALL_LIST = [ +IF_UP_DOWN_BRING_UP_CALL_LIST: list = [ ((["ifup", "eth0"],), {}), ((["ifup", "eth1"],), {}), ] -NETWORK_MANAGER_BRING_UP_CALL_LIST = [ +NETWORK_MANAGER_BRING_UP_CALL_LIST: list = [ ( ( [ @@ -230,7 +233,7 @@ def test_available(self, activator, available_calls, available_mocks): ), ] -NETWORKD_BRING_UP_CALL_LIST = [ +NETWORKD_BRING_UP_CALL_LIST: list = [ ((["ip", "link", "set", "up", "eth0"],), {}), ((["ip", "link", "set", "up", "eth1"],), {}), ((["systemctl", "restart", "systemd-networkd", "systemd-resolved"],), {}), @@ -286,17 +289,17 @@ def test_bring_up_all_interfaces_v2( assert call in expected_call_list -IF_UP_DOWN_BRING_DOWN_CALL_LIST = [ +IF_UP_DOWN_BRING_DOWN_CALL_LIST: list = [ ((["ifdown", "eth0"],), {}), ((["ifdown", "eth1"],), {}), ] -NETWORK_MANAGER_BRING_DOWN_CALL_LIST = [ +NETWORK_MANAGER_BRING_DOWN_CALL_LIST: list = [ ((["nmcli", "device", "disconnect", "eth0"],), {}), ((["nmcli", "device", "disconnect", "eth1"],), {}), ] -NETWORKD_BRING_DOWN_CALL_LIST = [ +NETWORKD_BRING_DOWN_CALL_LIST: list = [ ((["ip", "link", "set", "down", "eth0"],), {}), ((["ip", "link", "set", "down", "eth1"],), {}), ] diff --git a/tests/unittests/test_persistence.py b/tests/unittests/test_persistence.py index ec1152a91..8cc0d25a3 100644 --- a/tests/unittests/test_persistence.py +++ b/tests/unittests/test_persistence.py @@ -25,6 +25,7 @@ """ import pickle +from typing import List, Type from unittest import mock import pytest @@ -35,7 +36,7 @@ class _Collector(type): """Any class using this as a metaclass will be stored in test_classes.""" - test_classes = [] + test_classes: List[Type] = [] def __new__(cls, *args): new_cls = super().__new__(cls, *args) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_ssh_util.py similarity index 69% rename from tests/unittests/test_sshutil.py rename to tests/unittests/test_ssh_util.py index 445868354..6ce717389 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_ssh_util.py @@ -1,30 +1,27 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -from collections import namedtuple +import stat from functools import partial +from typing import NamedTuple +from unittest import mock from unittest.mock import patch +import pytest + from cloudinit import ssh_util, util -from cloudinit.temp_utils import mkdtemp -from tests.unittests import helpers as test_helpers - -# https://stackoverflow.com/questions/11351032/ -FakePwEnt = namedtuple( - "FakePwEnt", - [ - "pw_name", - "pw_passwd", - "pw_uid", - "pw_gid", - "pw_gecos", - "pw_dir", - "pw_shell", - ], -) -FakePwEnt.__new__.__defaults__ = tuple( - "UNSET_%s" % n for n in FakePwEnt._fields -) + +M_PATH = "cloudinit.ssh_util." + + +class FakePwEnt(NamedTuple): + pw_name: str = "UNSET_pw_name" + pw_passwd: str = "UNSET_w_passwd" + pw_uid: str = "UNSET_pw_uid" + pw_gid: str = "UNSET_pw_gid" + pw_gecos: str = "UNSET_pw_gecos" + pw_dir: str = "UNSET_pw_dir" + pw_shell: str = "UNSET_pw_shell" def mock_get_owner(updated_permissions, value): @@ -322,66 +319,40 @@ def mock_getpwnam(users, username): ) -class TestAuthKeyLineParser(test_helpers.CiTestCase): - def test_simple_parse(self): - # test key line with common 3 fields (keytype, base64, comment) - parser = ssh_util.AuthKeyLineParser() - for ktype in KEY_TYPES: - content = VALID_CONTENT[ktype] - comment = "user-%s@host" % ktype - line = " ".join( - ( - ktype, - content, - comment, - ) - ) - key = parser.parse(line) - - self.assertEqual(key.base64, content) - self.assertFalse(key.options) - self.assertEqual(key.comment, comment) - self.assertEqual(key.keytype, ktype) - - def test_parse_no_comment(self): - # test key line with key type and base64 only - parser = ssh_util.AuthKeyLineParser() - for ktype in KEY_TYPES: - content = VALID_CONTENT[ktype] - line = " ".join( - ( - ktype, - content, - ) - ) - key = parser.parse(line) - - self.assertEqual(key.base64, content) - self.assertFalse(key.options) - self.assertFalse(key.comment) - self.assertEqual(key.keytype, ktype) - - def test_parse_with_keyoptions(self): - # test key line with options in it - parser = ssh_util.AuthKeyLineParser() +class TestAuthKeyLineParser: + @pytest.mark.parametrize("with_options", [True, False]) + @pytest.mark.parametrize("with_comment", [True, False]) + @pytest.mark.parametrize("ktype", KEY_TYPES) + def test_parse(self, ktype, with_comment, with_options): + content = VALID_CONTENT[ktype] + comment = "user-%s@host" % ktype options = TEST_OPTIONS - for ktype in KEY_TYPES: - content = VALID_CONTENT[ktype] - comment = "user-%s@host" % ktype - line = " ".join( - ( - options, - ktype, - content, - comment, - ) - ) - key = parser.parse(line) - - self.assertEqual(key.base64, content) - self.assertEqual(key.options, options) - self.assertEqual(key.comment, comment) - self.assertEqual(key.keytype, ktype) + + line_args = [] + if with_options: + line_args.append(options) + line_args.extend( + [ + ktype, + content, + ] + ) + if with_comment: + line_args.append(comment) + line = " ".join(line_args) + + key = ssh_util.AuthKeyLineParser().parse(line) + + assert key.base64 == content + assert key.keytype == ktype + if with_options: + assert key.options == options + else: + assert key.options is None + if with_comment: + assert key.comment == comment + else: + assert key.comment == "" def test_parse_with_options_passed_in(self): # test key line with key type and base64 only @@ -391,30 +362,44 @@ def test_parse_with_options_passed_in(self): myopts = "no-port-forwarding,no-agent-forwarding" key = parser.parse("allowedopt" + " " + baseline) - self.assertEqual(key.options, "allowedopt") + assert key.options == "allowedopt" key = parser.parse("overridden_opt " + baseline, options=myopts) - self.assertEqual(key.options, myopts) + assert key.options == myopts def test_parse_invalid_keytype(self): parser = ssh_util.AuthKeyLineParser() key = parser.parse(" ".join(["badkeytype", VALID_CONTENT["rsa"]])) - self.assertFalse(key.valid()) + assert not key.valid() -class TestUpdateAuthorizedKeys(test_helpers.CiTestCase): - def test_new_keys_replace(self): +class TestUpdateAuthorizedKeys: + @pytest.mark.parametrize( + "new_entries", + [ + ( + [ + " ".join(("rsa", VALID_CONTENT["rsa"], "new_comment1")), + ] + ), + pytest.param( + [ + " ".join(("rsa", VALID_CONTENT["rsa"], "new_comment1")), + "xxx-invalid-thing1", + "xxx-invalid-blob2", + ], + id="skip-invalid-entries", + ), + ], + ) + def test_new_keys_replace(self, new_entries): """new entries with the same base64 should replace old.""" orig_entries = [ " ".join(("rsa", VALID_CONTENT["rsa"], "orig_comment1")), " ".join(("dsa", VALID_CONTENT["dsa"], "orig_comment2")), ] - new_entries = [ - " ".join(("rsa", VALID_CONTENT["rsa"], "new_comment1")), - ] - expected = "\n".join([new_entries[0], orig_entries[1]]) + "\n" parser = ssh_util.AuthKeyLineParser() @@ -423,100 +408,77 @@ def test_new_keys_replace(self): [parser.parse(p) for p in new_entries], ) - self.assertEqual(expected, found) - - def test_new_invalid_keys_are_ignored(self): - """new entries that are invalid should be skipped.""" - orig_entries = [ - " ".join(("rsa", VALID_CONTENT["rsa"], "orig_comment1")), - " ".join(("dsa", VALID_CONTENT["dsa"], "orig_comment2")), - ] - - new_entries = [ - " ".join(("rsa", VALID_CONTENT["rsa"], "new_comment1")), - "xxx-invalid-thing1", - "xxx-invalid-blob2", - ] - - expected = "\n".join([new_entries[0], orig_entries[1]]) + "\n" - - parser = ssh_util.AuthKeyLineParser() - found = ssh_util.update_authorized_keys( - [parser.parse(p) for p in orig_entries], - [parser.parse(p) for p in new_entries], - ) - - self.assertEqual(expected, found) - - -class TestParseSSHConfig(test_helpers.CiTestCase): - def setUp(self): - self.load_file_patch = patch("cloudinit.ssh_util.util.load_file") - self.load_file = self.load_file_patch.start() - self.isfile_patch = patch("cloudinit.ssh_util.os.path.isfile") - self.isfile = self.isfile_patch.start() - self.isfile.return_value = True - - def tearDown(self): - self.load_file_patch.stop() - self.isfile_patch.stop() - - def test_not_a_file(self): - self.isfile.return_value = False - self.load_file.side_effect = IOError - ret = ssh_util.parse_ssh_config("not a real file") - self.assertEqual([], ret) - - def test_empty_file(self): - self.load_file.return_value = "" - ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual([], ret) - - def test_comment_line(self): - comment_line = "# This is a comment" - self.load_file.return_value = comment_line + assert expected == found + + +@mock.patch(M_PATH + "util.load_file") +@mock.patch(M_PATH + "os.path.isfile") +class TestParseSSHConfig: + @pytest.mark.parametrize( + "is_file, file_content", + [ + pytest.param(True, ("",), id="empty-file"), + pytest.param(False, IOError, id="not-a-file"), + ], + ) + def test_dummy_file(self, m_is_file, m_load_file, is_file, file_content): + m_is_file.return_value = is_file + m_load_file.side_effect = file_content + ret = ssh_util.parse_ssh_config("notmatter") + assert [] == ret + + @pytest.mark.parametrize( + "file_content", + [ + pytest.param(["# This is a comment"], id="comment_line"), + pytest.param( + ["# This is a comment", "# This is another comment"], + id="two-comment_lines", + ), + ], + ) + def test_comment_line(self, m_is_file, m_load_file, file_content): + m_is_file.return_value = True + m_load_file.return_value = "\n".join(file_content) ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual(1, len(ret)) - self.assertEqual(comment_line, ret[0].line) + assert len(file_content) == len(ret) + assert file_content[0] == ret[0].line - def test_blank_lines(self): + def test_blank_lines(self, m_is_file, m_load_file): + m_is_file.return_value = True lines = ["", "\t", " "] - self.load_file.return_value = "\n".join(lines) + m_load_file.return_value = "\n".join(lines) ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual(len(lines), len(ret)) + assert len(lines) == len(ret) for line in ret: - self.assertEqual("", line.line) - - def test_lower_case_config(self): - self.load_file.return_value = "foo bar" - ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual(1, len(ret)) - self.assertEqual("foo", ret[0].key) - self.assertEqual("bar", ret[0].value) - - def test_upper_case_config(self): - self.load_file.return_value = "Foo Bar" - ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual(1, len(ret)) - self.assertEqual("foo", ret[0].key) - self.assertEqual("Bar", ret[0].value) - - def test_lower_case_with_equals(self): - self.load_file.return_value = "foo=bar" - ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual(1, len(ret)) - self.assertEqual("foo", ret[0].key) - self.assertEqual("bar", ret[0].value) - - def test_upper_case_with_equals(self): - self.load_file.return_value = "Foo=bar" + assert "" == line.line + + @pytest.mark.parametrize( + "file_content, expected_key, expected_value", + [ + pytest.param("foo bar", "foo", "bar", id="lower-case"), + pytest.param("Foo Bar", "foo", "Bar", id="upper-case"), + pytest.param("foo=bar", "foo", "bar", id="lower-case-with-equals"), + pytest.param("Foo=bar", "foo", "bar", id="upper-case-with-equals"), + ], + ) + def test_case_config( + self, + m_is_file, + m_load_file, + file_content, + expected_key, + expected_value, + ): + m_is_file.return_value = True + m_load_file.return_value = file_content ret = ssh_util.parse_ssh_config("some real file") - self.assertEqual(1, len(ret)) - self.assertEqual("foo", ret[0].key) - self.assertEqual("bar", ret[0].value) + assert 1 == len(ret) + assert expected_key == ret[0].key + assert expected_value == ret[0].value -class TestUpdateSshConfigLines(test_helpers.CiTestCase): +class TestUpdateSshConfigLines: """Test the update_ssh_config_lines method.""" exlines = [ @@ -529,24 +491,25 @@ class TestUpdateSshConfigLines(test_helpers.CiTestCase): pwauth = "PasswordAuthentication" def check_line(self, line, opt, val): - self.assertEqual(line.key, opt.lower()) - self.assertEqual(line.value, val) - self.assertIn(opt, str(line)) - self.assertIn(val, str(line)) - - def test_new_option_added(self): - """A single update of non-existing option.""" - lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) - result = ssh_util.update_ssh_config_lines(lines, {"MyKey": "MyVal"}) - self.assertEqual(["MyKey"], result) - self.check_line(lines[-1], "MyKey", "MyVal") - - def test_commented_out_not_updated_but_appended(self): - """Implementation does not un-comment and update lines.""" + assert line.key == opt.lower() + assert line.value == val + assert opt in str(line) + assert val in str(line) + + @pytest.mark.parametrize( + "key, value", + [ + pytest.param("MyKey", "MyVal", id="new_option_added"), + pytest.param( + pwauth, "no", id="commented_out_not_updated_but_appended" + ), + ], + ) + def test_update_ssh_config_lines(self, key, value): lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) - result = ssh_util.update_ssh_config_lines(lines, {self.pwauth: "no"}) - self.assertEqual([self.pwauth], result) - self.check_line(lines[-1], self.pwauth, "no") + result = ssh_util.update_ssh_config_lines(lines, {key: value}) + assert [key] == result + self.check_line(lines[-1], key, value) def test_option_without_value(self): """Implementation only accepts key-value pairs.""" @@ -554,14 +517,14 @@ def test_option_without_value(self): denyusers_opt = "DenyUsers" extended_exlines.append(denyusers_opt) lines = ssh_util.parse_ssh_config_lines(list(extended_exlines)) - self.assertNotIn(denyusers_opt, str(lines)) + assert denyusers_opt not in str(lines) def test_single_option_updated(self): """A single update should have change made and line updated.""" opt, val = ("UsePAM", "no") lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) result = ssh_util.update_ssh_config_lines(lines, {opt: val}) - self.assertEqual([opt], result) + assert [opt] == result self.check_line(lines[1], opt, val) def test_multiple_updates_with_add(self): @@ -574,7 +537,7 @@ def test_multiple_updates_with_add(self): } lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) result = ssh_util.update_ssh_config_lines(lines, updates) - self.assertEqual(set(["UsePAM", "NewOpt", "AcceptEnv"]), set(result)) + assert set(["UsePAM", "NewOpt", "AcceptEnv"]) == set(result) self.check_line(lines[3], "AcceptEnv", updates["AcceptEnv"]) def test_return_empty_if_no_changes(self): @@ -582,8 +545,8 @@ def test_return_empty_if_no_changes(self): updates = {"UsePAM": "yes"} lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) result = ssh_util.update_ssh_config_lines(lines, updates) - self.assertEqual([], result) - self.assertEqual(self.exlines, [str(line) for line in lines]) + assert [] == result + assert self.exlines == [str(line) for line in lines] def test_keycase_not_modified(self): """Original case of key should not be changed on update. @@ -591,109 +554,150 @@ def test_keycase_not_modified(self): updates = {"usepam": "no"} lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) result = ssh_util.update_ssh_config_lines(lines, updates) - self.assertEqual(["usepam"], result) - self.assertEqual("UsePAM no", str(lines[1])) + assert ["usepam"] == result + assert "UsePAM no" == str(lines[1]) -class TestUpdateSshConfig(test_helpers.CiTestCase): +class TestUpdateSshConfig: cfgdata = "\n".join(["#Option val", "MyKey ORIG_VAL", ""]) - def test_modified(self): - mycfg = self.tmp_path("ssh_config_1") + def test_modified(self, tmpdir): + mycfg = tmpdir.join("ssh_config_1") util.write_file(mycfg, self.cfgdata) ret = ssh_util.update_ssh_config({"MyKey": "NEW_VAL"}, mycfg) - self.assertTrue(ret) + assert True is ret found = util.load_file(mycfg) - self.assertEqual(self.cfgdata.replace("ORIG_VAL", "NEW_VAL"), found) + assert self.cfgdata.replace("ORIG_VAL", "NEW_VAL") == found # assert there is a newline at end of file (LP: #1677205) - self.assertEqual("\n", found[-1]) + assert "\n" == found[-1] - def test_not_modified(self): - mycfg = self.tmp_path("ssh_config_2") + def test_not_modified(self, tmpdir): + mycfg = tmpdir.join("ssh_config_2") util.write_file(mycfg, self.cfgdata) with patch("cloudinit.ssh_util.util.write_file") as m_write_file: ret = ssh_util.update_ssh_config({"MyKey": "ORIG_VAL"}, mycfg) - self.assertFalse(ret) - self.assertEqual(self.cfgdata, util.load_file(mycfg)) + assert False is ret + assert self.cfgdata == util.load_file(mycfg) m_write_file.assert_not_called() - -class TestBasicAuthorizedKeyParse(test_helpers.CiTestCase): - def test_user(self): - self.assertEqual( - ["/opt/bobby/keys"], - ssh_util.render_authorizedkeysfile_paths( - "/opt/%u/keys", "/home/bobby", "bobby" + def test_without_include(self, tmpdir): + mycfg = tmpdir.join("sshd_config") + cfg = "X Y" + util.write_file(mycfg, cfg) + assert ssh_util.update_ssh_config({"key": "value"}, mycfg) + assert "X Y\nkey value\n" == util.load_file(mycfg) + expected_conf_file = f"{mycfg}.d/50-cloud-init.conf" + assert not os.path.isfile(expected_conf_file) + + @pytest.mark.parametrize( + "cfg", + ["Include {mycfg}.d/*.conf", "Include {mycfg}.d/*.conf # comment"], + ) + def test_with_include(self, cfg, tmpdir): + mycfg = tmpdir.join("sshd_config") + util.write_file(mycfg, cfg.format(mycfg=mycfg)) + assert ssh_util.update_ssh_config({"key": "value"}, mycfg) + expected_conf_file = f"{mycfg}.d/50-cloud-init.conf" + assert os.path.isfile(expected_conf_file) + assert 0o600 == stat.S_IMODE(os.stat(expected_conf_file).st_mode) + assert "key value\n" == util.load_file(expected_conf_file) + + def test_with_commented_include(self, tmpdir): + mycfg = tmpdir.join("sshd_config") + cfg = f"# Include {mycfg}.d/*.conf" + util.write_file(mycfg, cfg) + assert ssh_util.update_ssh_config({"key": "value"}, mycfg) + assert f"{cfg}\nkey value\n" == util.load_file(mycfg) + expected_conf_file = f"{mycfg}.d/50-cloud-init.conf" + assert not os.path.isfile(expected_conf_file) + + def test_with_other_include(self, tmpdir): + mycfg = tmpdir.join("sshd_config") + cfg = f"Include other_{mycfg}.d/*.conf" + util.write_file(mycfg, cfg) + assert ssh_util.update_ssh_config({"key": "value"}, mycfg) + assert f"{cfg}\nkey value\n" == util.load_file(mycfg) + expected_conf_file = f"{mycfg}.d/50-cloud-init.conf" + assert not os.path.isfile(expected_conf_file) + assert not os.path.isfile(f"other_{mycfg}.d/50-cloud-init.conf") + + +class TestBasicAuthorizedKeyParse: + @pytest.mark.parametrize( + "value, homedir, username, expected_rendered", + [ + pytest.param( + "/opt/%u/keys", + "/home/bobby", + "bobby", + ["/opt/bobby/keys"], + id="user", ), - ) - - def test_user_file(self): - self.assertEqual( - ["/opt/bobby"], - ssh_util.render_authorizedkeysfile_paths( - "/opt/%u", "/home/bobby", "bobby" + pytest.param( + "/opt/%u", + "/home/bobby", + "bobby", + ["/opt/bobby"], + id="user_file", ), - ) - - def test_user_file2(self): - self.assertEqual( - ["/opt/bobby/bobby"], - ssh_util.render_authorizedkeysfile_paths( - "/opt/%u/%u", "/home/bobby", "bobby" + pytest.param( + "/opt/%u/%u", + "/home/bobby", + "bobby", + ["/opt/bobby/bobby"], + id="user_file_2", ), - ) - - def test_multiple(self): - self.assertEqual( - ["/keys/path1", "/keys/path2"], - ssh_util.render_authorizedkeysfile_paths( - "/keys/path1 /keys/path2", "/home/bobby", "bobby" + pytest.param( + "/keys/path1 /keys/path2", + "/home/bobby", + "bobby", + ["/keys/path1", "/keys/path2"], + id="multiple", ), - ) - - def test_multiple2(self): - self.assertEqual( - ["/keys/path1", "/keys/bobby"], - ssh_util.render_authorizedkeysfile_paths( - "/keys/path1 /keys/%u", "/home/bobby", "bobby" + pytest.param( + "/keys/path1 /keys/%u", + "/home/bobby", + "bobby", + ["/keys/path1", "/keys/bobby"], + id="multiple_2", ), - ) - - def test_relative(self): - self.assertEqual( - ["/home/bobby/.secret/keys"], - ssh_util.render_authorizedkeysfile_paths( - ".secret/keys", "/home/bobby", "bobby" + pytest.param( + ".secret/keys", + "/home/bobby", + "bobby", + ["/home/bobby/.secret/keys"], + id="relative", ), - ) - - def test_home(self): - self.assertEqual( - ["/homedirs/bobby/.keys"], - ssh_util.render_authorizedkeysfile_paths( - "%h/.keys", "/homedirs/bobby", "bobby" + pytest.param( + "%h/.keys", + "/homedirs/bobby", + "bobby", + ["/homedirs/bobby/.keys"], + id="home", ), - ) - - def test_all(self): - self.assertEqual( - [ - "/homedirs/bobby/.keys", - "/homedirs/bobby/.secret/keys", - "/keys/path1", - "/opt/bobby/keys", - ], - ssh_util.render_authorizedkeysfile_paths( + pytest.param( "%h/.keys .secret/keys /keys/path1 /opt/%u/keys", "/homedirs/bobby", "bobby", + [ + "/homedirs/bobby/.keys", + "/homedirs/bobby/.secret/keys", + "/keys/path1", + "/opt/bobby/keys", + ], + id="all", ), + ], + ) + def test_render_authorizedkeysfile_paths( + self, value, homedir, username, expected_rendered + ): + assert expected_rendered == ssh_util.render_authorizedkeysfile_paths( + value, homedir, username ) -class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): - tmp_d = mkdtemp() - +class TestMultipleSshAuthorizedKeysFile: def create_fake_users( self, names, @@ -703,15 +707,16 @@ def create_fake_users( m_get_permissions, m_getpwnam, users, + tmpdir, ): homes = [] - root = self.tmp_d + "/root" + root = str(tmpdir.join("root")) fpw = FakePwEnt(pw_name="root", pw_dir=root) users["root"] = fpw for name in names: - home = self.tmp_d + "/home/" + name + home = str(tmpdir.join("home", name)) fpw = FakePwEnt(pw_name=name, pw_dir=home) users[name] = fpw homes.append(home) @@ -725,29 +730,29 @@ def create_fake_users( return homes def create_user_authorized_file(self, home, filename, content_key, keys): - user_ssh_folder = "%s/.ssh" % home + user_ssh_folder = os.path.join(home, ".ssh") # /tmp/home//.ssh/authorized_keys = content_key - authorized_keys = self.tmp_path(filename, dir=user_ssh_folder) + authorized_keys = str(os.path.join(user_ssh_folder, filename)) util.write_file(authorized_keys, VALID_CONTENT[content_key]) keys[authorized_keys] = content_key return authorized_keys - def create_global_authorized_file(self, filename, content_key, keys): - authorized_keys = self.tmp_path(filename, dir=self.tmp_d) + def create_global_authorized_file( + self, filename, content_key, keys, tmpdir + ): + authorized_keys = str(tmpdir.join(filename)) util.write_file(authorized_keys, VALID_CONTENT[content_key]) keys[authorized_keys] = content_key return authorized_keys - def create_sshd_config(self, authorized_keys_files): - sshd_config = self.tmp_path("sshd_config", dir=self.tmp_d) + def create_sshd_config(self, authorized_keys_files, tmpdir): + sshd_config = str(tmpdir.join("sshd_config")) util.write_file( sshd_config, "AuthorizedKeysFile " + authorized_keys_files ) return sshd_config - def execute_and_check( - self, user, sshd_config, solution, keys, delete_keys=True - ): + def execute_and_check(self, user, sshd_config, solution, keys): (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( user, sshd_config ) diff --git a/tests/unittests/test_stages.py b/tests/unittests/test_stages.py index 9fa2e6299..7fde2bac6 100644 --- a/tests/unittests/test_stages.py +++ b/tests/unittests/test_stages.py @@ -11,31 +11,11 @@ from cloudinit.sources import NetworkConfigSource from cloudinit.util import write_file from tests.unittests.helpers import mock +from tests.unittests.util import TEST_INSTANCE_ID, FakeDataSource -TEST_INSTANCE_ID = "i-testing" M_PATH = "cloudinit.stages." -class FakeDataSource(sources.DataSource): - def __init__( - self, paths=None, userdata=None, vendordata=None, network_config="" - ): - super(FakeDataSource, self).__init__({}, None, paths=paths) - self.metadata = {"instance-id": TEST_INSTANCE_ID} - self.userdata_raw = userdata - self.vendordata_raw = vendordata - self._network_config = None - if network_config: # Permit for None value to setup attribute - self._network_config = network_config - - @property - def network_config(self): - return self._network_config - - def _get_data(self): - return True - - class TestInit: @pytest.fixture(autouse=True) def setup(self, tmpdir): diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index d7a721a28..218915ed6 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -1,7 +1,5 @@ # Copyright (C) 2020 Canonical Ltd. # -# Author: Daniel Watkins -# # This file is part of cloud-init. See LICENSE file for license information. """Upgrade testing for cloud-init. @@ -18,9 +16,16 @@ import pytest -from cloudinit.stages import _pkl_load +from cloudinit.sources import pkl_load +from cloudinit.sources.DataSourceAzure import DataSourceAzure +from cloudinit.sources.DataSourceNoCloud import DataSourceNoCloud from tests.unittests.helpers import resourceLocation +DSNAME_TO_CLASS = { + "Azure": DataSourceAzure, + "NoCloud": DataSourceNoCloud, +} + class TestUpgrade: @pytest.fixture( @@ -34,7 +39,26 @@ def previous_obj_pkl(self, request): Test implementations _must not_ modify the ``previous_obj_pkl`` which they are passed, as that will affect tests that run after them. """ - return _pkl_load(str(request.param)) + return pkl_load(str(request.param)) + + def test_pkl_load_defines_all_init_side_effect_attributes( + self, previous_obj_pkl + ): + """Any attrs as side-effects of __init__ exist in unpickled obj.""" + ds_class = DSNAME_TO_CLASS[previous_obj_pkl.dsname] + sys_cfg = previous_obj_pkl.sys_cfg + distro = previous_obj_pkl.distro + paths = previous_obj_pkl.paths + ds = ds_class(sys_cfg, distro, paths) + if ds.dsname == "NoCloud" and previous_obj_pkl.__dict__: + expected = ( + set({"seed_dirs"}), # LP: #1568150 handled with getattr checks + set(), + ) + else: + expected = (set(),) + missing_attrs = ds.__dict__.keys() - previous_obj_pkl.__dict__.keys() + assert missing_attrs in expected def test_networking_set_on_distro(self, previous_obj_pkl): """We always expect to have ``.networking`` on ``Distro`` objects.""" diff --git a/tests/unittests/test_url_helper.py b/tests/unittests/test_url_helper.py index a9b9a85fb..214e57279 100644 --- a/tests/unittests/test_url_helper.py +++ b/tests/unittests/test_url_helper.py @@ -15,6 +15,7 @@ NOT_FOUND, REDACTED, UrlError, + UrlResponse, dual_stack, oauth_headers, read_file_or_url, @@ -92,6 +93,18 @@ def test_read_file_or_url_str_from_url(self): self.assertEqual(result.contents, data) self.assertEqual(str(result), data.decode("utf-8")) + @httpretty.activate + def test_read_file_or_url_str_from_url_streamed(self): + """Test that str(result.contents) on url is text version of contents. + It should not be "b'data'", but just "'data'" """ + url = "http://hostname/path" + data = b"This is my url content\n" + httpretty.register_uri(httpretty.GET, url, data) + result = read_file_or_url(url, stream=True) + assert isinstance(result, UrlResponse) + self.assertEqual(result.contents, data) + self.assertEqual(str(result), data.decode("utf-8")) + @httpretty.activate def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): """Headers are redacted from logs but unredacted in requests.""" @@ -146,6 +159,7 @@ def request(cls, **kwargs): "User-Agent": "Cloud-Init/%s" % (version.version_string()) }, + "stream": False, }, kwargs, ) @@ -186,6 +200,7 @@ def test_read_file_or_url_passes_params_to_readurl( "ssl_details": {"cert_file": "/path/cert.pem"}, "headers_cb": "headers_cb", "exception_cb": "exception_cb", + "stream": True, } assert response == read_file_or_url(**params) @@ -222,6 +237,7 @@ def request(cls, **kwargs): % (version.version_string()) }, "timeout": request_timeout, + "stream": False, } if request_timeout is None: expected_kwargs.pop("timeout") @@ -282,7 +298,7 @@ class TestDualStack: """ @pytest.mark.parametrize( - "func," "addresses," "stagger_delay," "timeout," "expected_val,", + ["func", "addresses", "stagger_delay", "timeout", "expected_val"], [ # Assert order based on timeout (lambda x, _: x, ("one", "two"), 1, 1, "one"), @@ -346,12 +362,14 @@ def test_dual_stack( event.set() @pytest.mark.parametrize( - "func," - "addresses," - "stagger_delay," - "timeout," - "message," - "expected_exc", + [ + "func", + "addresses", + "stagger_delay", + "timeout", + "message", + "expected_exc", + ], [ ( lambda _a, _b: 1 / 0, @@ -370,7 +388,7 @@ def test_dual_stack( ZeroDivisionError, ), ( - lambda _a, _b: [][0], + lambda _a, _b: [][0], # pylint: disable=E0643 ("matter", "these"), 0, 1, @@ -479,7 +497,7 @@ def response_nowait(cls, _request): return (200, {"request-id": "0"}, cls.success) @pytest.mark.parametrize( - "addresses," "expected_address_index," "response,", + ["addresses", "expected_address_index", "response"], [ # Use timeout to test ordering happens as expected ((ADDR1, SLEEP1), 0, "SUCCESS"), diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index bcb637871..0b297ef1e 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -15,13 +15,14 @@ import tempfile from collections import deque from textwrap import dedent -from typing import Tuple from unittest import mock import pytest import yaml from cloudinit import importer, subp, util +from cloudinit.helpers import Paths +from cloudinit.sources import DataSourceHostname from cloudinit.subp import SubpResult from tests.unittests import helpers from tests.unittests.helpers import CiTestCase @@ -321,23 +322,24 @@ BUG_REPORT_URL="https://github.com/vmware/photon/issues" """ - -class FakeCloud(object): - def __init__(self, hostname, fqdn): - self.hostname = hostname - self.fqdn = fqdn - self.calls = [] - - def get_hostname(self, fqdn=None, metadata_only=None): - myargs = {} - if fqdn is not None: - myargs["fqdn"] = fqdn - if metadata_only is not None: - myargs["metadata_only"] = metadata_only - self.calls.append(myargs) - if fqdn: - return self.fqdn - return self.hostname +OS_RELEASE_OPENMANDRIVA = dedent( + """\ + NAME="OpenMandriva Lx"\n + VERSION="4.90 (Nickel) Cooker"\n + ID="openmandriva"\n + VERSION_ID="4.90"\n + PRETTY_NAME="OpenMandriva Lx 4.90 (Nickel) Cooker"\n + BUILD_ID="20220606.19"\n + VERSION_CODENAME="nickel"\n + ANSI_COLOR="1;43"\n + LOGO="openmandriva"\n + CPE_NAME="cpe:/o:openmandriva:openmandriva_lx:4.90"\n + HOME_URL="http://openmandriva.org/"\n + BUG_REPORT_URL="http://issues.openmandriva.org/"\n + SUPPORT_URL="https://forum.openmandriva.org"\n + PRIVACY_POLICY_URL="https://www.openmandriva.org/tos"\n +""" +) class TestUtil: @@ -443,6 +445,23 @@ def test_read_conf_with_confd_no_permissions( assert [mock.call(confd_fn)] == m_read_confd.call_args_list assert [expected_call] == m_mergemanydict.call_args_list + @pytest.mark.parametrize("custom_cloud_dir", [True, False]) + @mock.patch(M_PATH + "os.path.isfile", return_value=True) + @mock.patch(M_PATH + "os.path.isdir", return_value=True) + def test_fetch_ssl_details( + self, m_isdir, m_isfile, custom_cloud_dir, tmpdir + ): + cloud_dir = "/var/lib/cloud" + if custom_cloud_dir: + cloud_dir = tmpdir.join("cloud") + cert = os.path.join(cloud_dir, "instance", "data", "ssl", "cert.pem") + key = os.path.join(cloud_dir, "instance", "data", "ssl", "key.pem") + + paths = Paths({"cloud_dir": cloud_dir}) + ssl_details = util.fetch_ssl_details(paths) + assert {"cert_file": cert, "key_file": key} == ssl_details + assert 2 == m_isdir.call_count == m_isfile.call_count + class TestSymlink(CiTestCase): def test_sym_link_simple(self): @@ -552,7 +571,7 @@ def test_supports_comments(self): class TestGetHostnameFqdn(CiTestCase): def test_get_hostname_fqdn_from_only_cfg_fqdn(self): """When cfg only has the fqdn key, derive hostname and fqdn from it.""" - hostname, fqdn = util.get_hostname_fqdn( + hostname, fqdn, _ = util.get_hostname_fqdn( cfg={"fqdn": "myhost.domain.com"}, cloud=None ) self.assertEqual("myhost", hostname) @@ -560,7 +579,7 @@ def test_get_hostname_fqdn_from_only_cfg_fqdn(self): def test_get_hostname_fqdn_from_cfg_fqdn_and_hostname(self): """When cfg has both fqdn and hostname keys, return them.""" - hostname, fqdn = util.get_hostname_fqdn( + hostname, fqdn, _ = util.get_hostname_fqdn( cfg={"fqdn": "myhost.domain.com", "hostname": "other"}, cloud=None ) self.assertEqual("other", hostname) @@ -568,7 +587,7 @@ def test_get_hostname_fqdn_from_cfg_fqdn_and_hostname(self): def test_get_hostname_fqdn_from_cfg_hostname_with_domain(self): """When cfg has only hostname key which represents a fqdn, use that.""" - hostname, fqdn = util.get_hostname_fqdn( + hostname, fqdn, _ = util.get_hostname_fqdn( cfg={"hostname": "myhost.domain.com"}, cloud=None ) self.assertEqual("myhost", hostname) @@ -576,37 +595,48 @@ def test_get_hostname_fqdn_from_cfg_hostname_with_domain(self): def test_get_hostname_fqdn_from_cfg_hostname_without_domain(self): """When cfg has a hostname without a '.' query cloud.get_hostname.""" - mycloud = FakeCloud("cloudhost", "cloudhost.mycloud.com") - hostname, fqdn = util.get_hostname_fqdn( - cfg={"hostname": "myhost"}, cloud=mycloud + cloud = mock.MagicMock() + cloud.get_hostname.return_value = DataSourceHostname( + "cloudhost.mycloud.com", False + ) + hostname, fqdn, _ = util.get_hostname_fqdn( + cfg={"hostname": "myhost"}, cloud=cloud ) self.assertEqual("myhost", hostname) self.assertEqual("cloudhost.mycloud.com", fqdn) - self.assertEqual( - [{"fqdn": True, "metadata_only": False}], mycloud.calls - ) + assert [ + mock.call(fqdn=True, metadata_only=False) + ] == cloud.get_hostname.call_args_list def test_get_hostname_fqdn_from_without_fqdn_or_hostname(self): """When cfg has neither hostname nor fqdn cloud.get_hostname.""" - mycloud = FakeCloud("cloudhost", "cloudhost.mycloud.com") - hostname, fqdn = util.get_hostname_fqdn(cfg={}, cloud=mycloud) + cloud = mock.MagicMock() + cloud.get_hostname.side_effect = ( + DataSourceHostname("cloudhost.mycloud.com", False), + DataSourceHostname("cloudhost", False), + ) + hostname, fqdn, _ = util.get_hostname_fqdn(cfg={}, cloud=cloud) self.assertEqual("cloudhost", hostname) self.assertEqual("cloudhost.mycloud.com", fqdn) - self.assertEqual( - [{"fqdn": True, "metadata_only": False}, {"metadata_only": False}], - mycloud.calls, - ) + assert [ + mock.call(fqdn=True, metadata_only=False), + mock.call(metadata_only=False), + ] == cloud.get_hostname.call_args_list def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self): """Calls to cloud.get_hostname pass the metadata_only parameter.""" - mycloud = FakeCloud("cloudhost", "cloudhost.mycloud.com") - _hn, _fqdn = util.get_hostname_fqdn( - cfg={}, cloud=mycloud, metadata_only=True + cloud = mock.MagicMock() + cloud.get_hostname.side_effect = ( + DataSourceHostname("cloudhost.mycloud.com", False), + DataSourceHostname("cloudhost", False), ) - self.assertEqual( - [{"fqdn": True, "metadata_only": True}, {"metadata_only": True}], - mycloud.calls, + _hn, _fqdn, _def_hostname = util.get_hostname_fqdn( + cfg={}, cloud=cloud, metadata_only=True ) + assert [ + mock.call(fqdn=True, metadata_only=True), + mock.call(metadata_only=True), + ] == cloud.get_hostname.call_args_list class TestBlkid(CiTestCase): @@ -754,9 +784,7 @@ def test_subp_exception_raises_to_caller(self, m_subp): @mock.patch("os.path.exists") class TestGetLinuxDistro(CiTestCase): def setUp(self): - # python2 has no lru_cache, and therefore, no cache_clear() - if hasattr(util.get_linux_distro, "cache_clear"): - util.get_linux_distro.cache_clear() + util.get_linux_distro.cache_clear() @classmethod def os_release_exists(self, path): @@ -1027,6 +1055,14 @@ def test_get_linux_photon_os_release(self, m_os_release, m_path_exists): dist = util.get_linux_distro() self.assertEqual(("photon", "4.0", "VMware Photon OS/Linux"), dist) + @mock.patch(M_PATH + "load_file") + def test_get_linux_openmandriva(self, m_os_release, m_path_exists): + """Verify we get the correct name and machine arch on OpenMandriva""" + m_os_release.return_value = OS_RELEASE_OPENMANDRIVA + m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists + dist = util.get_linux_distro() + self.assertEqual(("openmandriva", "4.90", "nickel"), dist) + @mock.patch("platform.system") @mock.patch("platform.dist", create=True) def test_get_linux_distro_no_data( @@ -1141,21 +1177,11 @@ def test_is_lxd_false_when_sock_device_absent(self, m_exists): class TestReadCcFromCmdline: - - random_string: Tuple - - if hasattr(pytest, "param"): - random_string = pytest.param( - CiTestCase.random_string(), None, id="random_string" - ) - else: - random_string = (CiTestCase.random_string(), None) - @pytest.mark.parametrize( "cmdline,expected_cfg", [ # Return None if cmdline has no cc:end_cc content. - random_string, + pytest.param(CiTestCase.random_string(), None, id="random_string"), # Return None if YAML content is empty string. ("foo cc: end_cc bar", None), # Return expected dictionary without trailing end_cc marker. @@ -2055,7 +2081,8 @@ def test_logs_go_to_console_by_default(self): self._createConsole(self.root) logged_string = "something very important" util.multi_log(logged_string) - self.assertEqual(logged_string, open("/dev/console").read()) + with open("/dev/console") as f: + self.assertEqual(logged_string, f.read()) def test_logs_dont_go_to_stdout_if_console_exists(self): self._createConsole(self.root) @@ -2437,6 +2464,47 @@ def test_get_proc_ppid(self): my_ppid = os.getppid() self.assertEqual(my_ppid, util.get_proc_ppid(my_pid)) + def test_get_proc_ppid_mocked(self): + for ppid, proc_data in ( + ( + 0, + "1 (systemd) S 0 1 1 0 -1 4194560 112664 14612195 153 18014" + "274 237 756828 152754 20 0 1 0 3 173809664 3736" + "18446744073709551615 1 1 0 0 0 0 671173123 4096 1260 0 0 0 17" + "8 0 0 0 0 123974 0 0 0 0 0 0 0 0", + ), + ( + 180771, + "180781 ([pytest-xdist r) R 180771 180598 167240 34825 " + "180598 4194304 128712 7570 0 0 1061 34 8 1 20 0 2 0 6551540 " + "351993856 25173 18446744073709551615 93907896635392 " + "93907899455533 140725724279536 0 0 0 0 16781312 17642 0 0 0 " + "17 1 0 0 0 0 0 93907901810800 93907902095288 93907928788992 " + "140725724288007 140725724288074 140725724288074 " + "140725724291047 0", + ), + ( + 5620, + "8723 (Utility Process) S 5620 5191 5191 0 -1 4194304 3219 " + "0 50 0 1045 431 0 0 20 0 3 0 9007 220585984 8758 " + "18446744073709551615 94469734690816 94469735319392 " + "140728350183632 0 0 0 0 69634 1073745144 0 0 0 17 10 0 0 0 0 " + "0 94469735327152 94469735331056 94469763170304 " + "140728350189012 140728350189221 140728350189221 " + "140728350195661 0", + ), + ( + 4946, + "4947 ((sd-pam)) S 4946 4946 4946 0 -1 1077936448 54 0 0 0 " + "0 0 0 0 20 0 1 0 4136 175616000 1394 18446744073709551615 1 1" + "0 0 0 0 0 4096 0 0 0 0 17 8 0 0 0 0 0 0 0 0 0 0 0 0 0", + ), + ): + with mock.patch( + "cloudinit.util.load_file", return_value=proc_data + ): + assert ppid == util.get_proc_ppid("mocked") + class TestKernelVersion: """test kernel version function""" @@ -2589,4 +2657,59 @@ def test_find_devs_with_dragonflybsd( assert devlist == expected_devlist -# vi: ts=4 expandtab +class TestVersion: + @pytest.mark.parametrize( + ("v1", "v2", "eq"), + ( + ("3.1.0", "3.1.0", True), + ("3.1.0", "3.1.1", False), + ("3.1", "3.1.0.0", False), + ), + ) + def test_eq(self, v1, v2, eq): + if eq: + assert util.Version.from_str(v1) == util.Version.from_str(v2) + if not eq: + assert util.Version.from_str(v1) != util.Version.from_str(v2) + + @pytest.mark.parametrize( + ("v1", "v2", "gt"), + ( + ("3.1.0", "3.1.0", False), + ("3.1.0", "3.1.1", False), + ("3.1", "3.1.0.0", False), + ("3.1.0.0", "3.1", True), + ("3.1.1", "3.1.0", True), + ), + ) + def test_gt(self, v1, v2, gt): + if gt: + assert util.Version.from_str(v1) > util.Version.from_str(v2) + if not gt: + assert util.Version.from_str(v1) < util.Version.from_str( + v2 + ) or util.Version.from_str(v1) == util.Version.from_str(v2) + + @pytest.mark.parametrize( + ("str_ver", "cls_ver"), + ( + ( + "0.0.0.0", + util.Version(0, 0, 0, 0), + ), + ( + "1.0.0.0", + util.Version(1, 0, 0, 0), + ), + ( + "1.0.2.0", + util.Version(1, 0, 2, 0), + ), + ( + "9.8.2.0", + util.Version(9, 8, 2, 0), + ), + ), + ) + def test_from_str(self, str_ver, cls_ver): + assert util.Version.from_str(str_ver) == cls_ver diff --git a/tests/unittests/util.py b/tests/unittests/util.py index f57a3d258..4635ca3fa 100644 --- a/tests/unittests/util.py +++ b/tests/unittests/util.py @@ -1,9 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + from cloudinit import cloud, distros, helpers +from cloudinit.sources import DataSource, DataSourceHostname from cloudinit.sources.DataSourceNone import DataSourceNone -def get_cloud(distro=None, paths=None, sys_cfg=None, metadata=None): +def get_cloud( + distro=None, paths=None, sys_cfg=None, metadata=None, mocked_distro=False +): """Obtain a "cloud" that can be used for testing. Modules take a 'cloud' parameter to call into things that are @@ -17,6 +22,8 @@ def get_cloud(distro=None, paths=None, sys_cfg=None, metadata=None): sys_cfg = sys_cfg or {} cls = distros.fetch(distro) if distro else MockDistro mydist = cls(distro, sys_cfg, paths) + if mocked_distro: + mydist = mock.Mock(wraps=mydist) myds = DataSourceTesting(sys_cfg, mydist, paths) if metadata: myds.metadata.update(metadata) @@ -37,7 +44,7 @@ class concreteCls(abclass): class DataSourceTesting(DataSourceNone): def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): - return "hostname" + return DataSourceHostname("hostname", False) def persist_instance_data(self): return True @@ -140,3 +147,32 @@ def package_command(self, command, args=None, pkgs=None): def update_package_sources(self): return (True, "yay") + + +TEST_INSTANCE_ID = "i-testing" + + +class FakeDataSource(DataSource): + def __init__( + self, + userdata=None, + vendordata=None, + vendordata2=None, + network_config="", + paths=None, + ): + DataSource.__init__(self, {}, None, paths=paths) + self.metadata = {"instance-id": TEST_INSTANCE_ID} + self.userdata_raw = userdata + self.vendordata_raw = vendordata + self.vendordata2_raw = vendordata2 + self._network_config = None + if network_config: # Permit for None value to setup attribute + self._network_config = network_config + + @property + def network_config(self): + return self._network_config + + def _get_data(self): + return True diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index cd7efbd45..271a47101 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -5,19 +5,21 @@ akutz AlexBaranowski Aman306 andgein +andrew-lee-metaswitch andrewbogott andrewlukoshko -andrew-lee-metaswitch antonyc aswinrajamannar beantaxi beezly +berolinux bipinbachhao BirknerAlex bmhughes candlerb cawamata cclauss +chifac08 chrislalos ciprianbadescu citrus-it @@ -25,6 +27,7 @@ cjp256 Conan-Kudo cvstealth dankenigsberg +david-caro ddymko dermotbradley dhensby @@ -35,6 +38,7 @@ emmanuelthome eslerm esposem GabrielNagy +garzdin giggsoff hamalq holmanb @@ -54,10 +58,12 @@ kallioli klausenbusk KsenijaS landon912 +linitio lkundrak lucasmoura lucendio lungj +magnetikonline mal mamercad manuelisimo @@ -68,6 +74,7 @@ megian michaelrommel mitechie nazunalika +netcho nicolasbock nishigori olivierlemasle @@ -75,20 +82,27 @@ omBratteng onitake Oursin qubidt +RedKrieg renanrodrigo rhansen riedel +rongz609 +SadeghHayeri sarahwzadara +scorpion44 +shaardie shi2wei3 slingamn slyon smoser sshedi +sstallion stappersg +stefanor steverweber t-8ch -TheRealFalcon taoyama +TheRealFalcon thetoolsmith timothegenzmer tnt-dev @@ -103,4 +117,5 @@ wschoot xiachen-rh xnox yangzz-97 +yawkat zhuzaifangxuele diff --git a/tools/read-version b/tools/read-version index 563a23bcf..32f169707 100755 --- a/tools/read-version +++ b/tools/read-version @@ -11,19 +11,11 @@ if "avoid-pep8-E402-import-not-top-of-file": from cloudinit import version as ci_version -def tiny_p(cmd, capture=True): - # python 2.6 doesn't have check_output - stdout = subprocess.PIPE +def tiny_p(cmd): stderr = subprocess.PIPE - sp = subprocess.Popen(cmd, stdout=stdout, - stderr=stderr, stdin=None, - universal_newlines=True) - (out, err) = sp.communicate() - ret = sp.returncode - if ret not in [0]: - raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" % - (cmd, ret, out, err)) - return out + return subprocess.check_output( + cmd, stderr=stderr, stdin=None, universal_newlines=True + ) def which(program): diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index 176df36b7..eae83217d 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -25,6 +25,7 @@ def main(): "netbsd", "openbsd", "openEuler", + "openmandriva", "photon", "rhel", "suse",