From 8907f3f0cc5e6710b2b889961029e31a067579e7 Mon Sep 17 00:00:00 2001 From: paulober Date: Thu, 17 Oct 2024 09:33:51 +0100 Subject: [PATCH 01/20] Added Raspberry Pi OS support This commit adds support for the Raspberry Pi OS debian distribution. It includes a distro definition, 3 modules and integration into other systems like config generation. Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- .gitignore | 1 + cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_apt_pipelining.py | 2 +- cloudinit/config/cc_byobu.py | 2 +- cloudinit/config/cc_ca_certs.py | 11 +- cloudinit/config/cc_ntp.py | 6 + cloudinit/config/cc_rpi_connect.py | 47 ++++ cloudinit/config/cc_rpi_interfaces.py | 180 ++++++++++++++ cloudinit/config/cc_rpi_userdata.py | 228 ++++++++++++++++++ cloudinit/config/cc_ssh_import_id.py | 2 +- .../schemas/schema-cloud-config-v1.json | 111 +++++++++ cloudinit/distros/__init__.py | 2 +- cloudinit/distros/raspberry-pi-os.py | 45 ++++ cloudinit/util.py | 3 + config/cloud.cfg.tmpl | 45 +++- doc/module-docs/cc_rpi_connect/data.yaml | 13 + doc/module-docs/cc_rpi_connect/example1.yaml | 2 + doc/module-docs/cc_rpi_interfaces/data.yaml | 19 ++ .../cc_rpi_interfaces/example1.yaml | 4 + .../cc_rpi_interfaces/example2.yaml | 3 + .../cc_rpi_interfaces/example3.yaml | 6 + .../cc_rpi_interfaces/example4.yaml | 8 + doc/module-docs/cc_rpi_userdata/data.yaml | 26 ++ doc/module-docs/cc_rpi_userdata/example1.yaml | 2 + doc/module-docs/cc_rpi_userdata/example2.yaml | 3 + doc/module-docs/cc_rpi_userdata/example3.yaml | 4 + doc/rtd/reference/availability.rst | 1 + doc/rtd/reference/modules.rst | 6 + doc/rtd/reference/network-config.rst | 8 +- pyproject.toml | 8 + systemd/cloud-init-local.service.tmpl | 4 +- systemd/cloud-init-main.service.tmpl | 3 +- systemd/cloud-init-network.service.tmpl | 14 +- tests/unittests/config/test_cc_rpi_connect.py | 76 ++++++ .../config/test_cc_rpi_interfaces.py | 124 ++++++++++ .../unittests/config/test_cc_rpi_userdata.py | 93 +++++++ tests/unittests/config/test_schema.py | 3 + tests/unittests/test_render_template.py | 6 + tools/render-template | 1 + 39 files changed, 1102 insertions(+), 22 deletions(-) create mode 100644 cloudinit/config/cc_rpi_connect.py create mode 100644 cloudinit/config/cc_rpi_interfaces.py create mode 100644 cloudinit/config/cc_rpi_userdata.py create mode 100644 cloudinit/distros/raspberry-pi-os.py create mode 100644 doc/module-docs/cc_rpi_connect/data.yaml create mode 100644 doc/module-docs/cc_rpi_connect/example1.yaml create mode 100644 doc/module-docs/cc_rpi_interfaces/data.yaml create mode 100644 doc/module-docs/cc_rpi_interfaces/example1.yaml create mode 100644 doc/module-docs/cc_rpi_interfaces/example2.yaml create mode 100644 doc/module-docs/cc_rpi_interfaces/example3.yaml create mode 100644 doc/module-docs/cc_rpi_interfaces/example4.yaml create mode 100644 doc/module-docs/cc_rpi_userdata/data.yaml create mode 100644 doc/module-docs/cc_rpi_userdata/example1.yaml create mode 100644 doc/module-docs/cc_rpi_userdata/example2.yaml create mode 100644 doc/module-docs/cc_rpi_userdata/example3.yaml create mode 100644 tests/unittests/config/test_cc_rpi_connect.py create mode 100644 tests/unittests/config/test_cc_rpi_interfaces.py create mode 100644 tests/unittests/config/test_cc_rpi_userdata.py diff --git a/.gitignore b/.gitignore index c9bc3823ce5..7d671fb9f21 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ srpm/ stage tags tests/integration_tests/user_settings.py + diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 853e529635c..d22fde88a37 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -41,7 +41,7 @@ meta: MetaSchema = { "id": "cc_apt_configure", - "distros": ["ubuntu", "debian"], + "distros": ["ubuntu", "debian", "raspberry-pi-os"], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], } diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index eacecd171b8..1fb9933bf64 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -28,7 +28,7 @@ meta: MetaSchema = { "id": "cc_apt_pipelining", - "distros": ["ubuntu", "debian"], + "distros": ["ubuntu", "debian", "raspberry-pi-os"], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["apt_pipelining"], } diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 86cc4bda4db..5e9a9a27275 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -21,7 +21,7 @@ meta: MetaSchema = { "id": "cc_byobu", - "distros": ["ubuntu", "debian"], + "distros": ["ubuntu", "debian", "raspberry-pi-os"], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], } diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index c508cc930d8..67250a05645 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -84,6 +84,7 @@ "alpine", "debian", "fedora", + "raspberry-pi-os", "rhel", "opensuse", "opensuse-microos", @@ -158,10 +159,16 @@ def disable_default_ca_certs(distro_name, distro_cfg): """ if distro_name in ["rhel", "photon"]: remove_default_ca_certs(distro_cfg) - elif distro_name in ["alpine", "aosc", "debian", "ubuntu"]: + elif distro_name in [ + "alpine", + "aosc", + "debian", + "raspberry-pi-os", + "ubuntu", + ]: disable_system_ca_certs(distro_cfg) - if distro_name in ["debian", "ubuntu"]: + if distro_name in ["debian", "raspberry-pi-os", "ubuntu"]: debconf_sel = ( "ca-certificates ca-certificates/trust_new_crts " + "select no" ) diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 0501a89a445..a8f1defbfe3 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -45,6 +45,7 @@ "opensuse-tumbleweed", "opensuse-leap", "photon", + "raspberry-pi-os", "rhel", "rocky", "sle_hpc", @@ -211,6 +212,11 @@ "confpath": "/etc/systemd/timesyncd.conf", }, }, + "raspberry-pi-os": { + "chrony": { + "confpath": "/etc/chrony/chrony.conf", + }, + }, "rhel": { "ntp": { "service_name": "ntpd", diff --git a/cloudinit/config/cc_rpi_connect.py b/cloudinit/config/cc_rpi_connect.py new file mode 100644 index 00000000000..09e6aa0c301 --- /dev/null +++ b/cloudinit/config/cc_rpi_connect.py @@ -0,0 +1,47 @@ +# Copyright (C) 2024, Raspberry Pi Ltd. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import subp +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema +from cloudinit.settings import PER_INSTANCE +import logging + + +LOG = logging.getLogger(__name__) +ENABLE_RPI_CONNECT_KEY = "enable_rpi_connect" + +meta: MetaSchema = { + "id": "cc_rpi_connect", + "distros": ["raspberry-pi-os"], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": [ENABLE_RPI_CONNECT_KEY], +} + + +def configure_rpi_connect(enable: bool) -> None: + LOG.debug(f"Configuring rpi-connect: {enable}") + + num = 0 if enable else 1 + + try: + subp.subp(["/usr/bin/raspi-config", "do_rpi_connect", str(num)]) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure rpi-connect: %s", e) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + if ENABLE_RPI_CONNECT_KEY in cfg: + # expect it to be a dictionary + enable = cfg[ENABLE_RPI_CONNECT_KEY] + + if isinstance(enable, bool): + configure_rpi_connect(enable) + else: + LOG.warning( + "Invalid value for %s: %s", ENABLE_RPI_CONNECT_KEY, enable + ) diff --git a/cloudinit/config/cc_rpi_interfaces.py b/cloudinit/config/cc_rpi_interfaces.py new file mode 100644 index 00000000000..d8fecf014b7 --- /dev/null +++ b/cloudinit/config/cc_rpi_interfaces.py @@ -0,0 +1,180 @@ +# Copyright (C) 2024, Raspberry Pi Ltd. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import subp +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema +from cloudinit.settings import PER_INSTANCE +import logging + + +LOG = logging.getLogger(__name__) +RPI_INTERFACES_KEY = "rpi_interfaces" +SUPPORTED_INTERFACES = { + "spi": "do_spi", + "i2c": "do_i2c", + "serial": "do_serial", + "onewire": "do_onewire", + "remote_gpio": "do_rgpio", + "ssh": "enable_ssh", +} +RASPI_CONFIG_SERIAL_CONS_FN = "do_serial_cons" +RASPI_CONFIG_SERIAL_HW_FN = "do_serial_hw" + +meta: MetaSchema = { + "id": "cc_rpi_interfaces", + "distros": ["raspberry-pi-os"], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": [RPI_INTERFACES_KEY], +} + + +# TODO: test +def require_reboot(cfg: Config) -> None: + cfg["power_state"] = cfg.get("power_state", {}) + cfg["power_state"]["mode"] = cfg["power_state"].get("mode", "reboot") + cfg["power_state"]["condition"] = True + + +def is_pifive() -> bool: + try: + subp.subp(["/usr/bin/raspi-config", "nonint", "is_pifive"]) + return True + except subp.ProcessExecutionError: + return False + + +def configure_serial_interface(cfg: dict | bool, instCfg: Config) -> None: + enable_console = False + enable_hw = False + + if isinstance(cfg, dict): + enable_console = cfg.get("console", False) + enable_hw = cfg.get("hardware", False) + elif isinstance(cfg, bool): + # default to enabling console as if < pi5 + # this will also enable the hardware + enable_console = cfg + + if not is_pifive() and enable_console: + # only pi5 has 2 usable UARTs + # on other models, enabling the console + # will also block the other UART + enable_hw = True + + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + RASPI_CONFIG_SERIAL_CONS_FN, + str(0 if enable_console else 1), + ] + ) + + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + RASPI_CONFIG_SERIAL_HW_FN, + str(0 if enable_hw else 1), + ] + ) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure serial hardware: %s", e) + + require_reboot(instCfg) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure serial console: %s", e) + + +def enable_ssh(cfg: Config, enable: bool) -> None: + if not enable: + return + + try: + subp.subp( + [ + "/usr/lib/raspberry-pi-sys-mods/imager_custom", + SUPPORTED_INTERFACES["ssh"], + ] + ) + require_reboot(cfg) + except subp.ProcessExecutionError as e: + LOG.error("Failed to enable ssh: %s", e) + + +def configure_interface(iface: str, enable: bool) -> None: + assert ( + iface in SUPPORTED_INTERFACES.keys() and iface != "serial" + ), f"Unsupported interface: {iface}" + + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + SUPPORTED_INTERFACES[iface], + str(0 if enable else 1), + ] + ) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure %s: %s", iface, e) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + if RPI_INTERFACES_KEY not in cfg: + return + elif not isinstance(cfg[RPI_INTERFACES_KEY], dict): + LOG.warning( + "Invalid value for %s: %s", + RPI_INTERFACES_KEY, + cfg[RPI_INTERFACES_KEY], + ) + return + elif not cfg[RPI_INTERFACES_KEY]: + LOG.debug("Empty value for %s. Skipping...", RPI_INTERFACES_KEY) + return + + # check for supported ARM interfaces + for key in cfg[RPI_INTERFACES_KEY]: + if key not in SUPPORTED_INTERFACES.keys(): + LOG.warning("Invalid key for %s: %s", RPI_INTERFACES_KEY, key) + continue + + enable = cfg[RPI_INTERFACES_KEY][key] + + if key == "serial": + if not isinstance(enable, dict) and not isinstance(enable, bool): + LOG.warning( + "Invalid value for %s.%s: %s", + RPI_INTERFACES_KEY, + key, + enable, + ) + else: + configure_serial_interface(enable, cfg) + continue + elif key == "ssh": + if not isinstance(enable, bool): + LOG.warning( + "Invalid value for %s.%s: %s", + RPI_INTERFACES_KEY, + key, + enable, + ) + else: + enable_ssh(cfg, enable) + continue + + if isinstance(enable, bool): + configure_interface(key, enable) + else: + LOG.warning( + "Invalid value for %s.%s: %s", RPI_INTERFACES_KEY, key, enable + ) diff --git a/cloudinit/config/cc_rpi_userdata.py b/cloudinit/config/cc_rpi_userdata.py new file mode 100644 index 00000000000..2c19f711187 --- /dev/null +++ b/cloudinit/config/cc_rpi_userdata.py @@ -0,0 +1,228 @@ +# Copyright (C) 2024, Raspberry Pi Ltd. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import subp +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema +from cloudinit.settings import PER_ALWAYS +import logging +import os +import subprocess +import time + +LOG = logging.getLogger(__name__) +DISABLE_PIWIZ_KEY = "disable_piwiz" +RPI_USERCONF_KEY = "rpi_userconf" +USERCONF_SERVICE_TTY = "/dev/tty8" +MODULE_DEACTIVATION_FILE = "/var/lib/userconf-pi/deactivate" + +meta: MetaSchema = { + "id": "cc_rpi_userdata", + "distros": ["raspberry-pi-os"], + # Run every boot to trigger setup wizard even when no settings + "frequency": PER_ALWAYS, + # "activate_by_schema_keys": [DISABLE_PIWIZ_KEY, RPI_USERCONF_KEY], + # When provided it would only start the module + # when the keys are present in the configuration + "activate_by_schema_keys": [], +} + + +def bool_to_str(value: bool | None) -> str: + return "Yes" if value else "No" + + +def get_fwloc_or_default() -> str: + fwloc = None + try: + # Run the command and capture the output + fwloc = subp.subp( + ["/usr/lib/raspberrypi-sys-mods/get_fw_loc"], decode="strict" + ).stdout.strip() + + # If the output is empty, set the default value + if not fwloc: + fwloc = "/boot" + except subp.ProcessExecutionError: + # If the command fails, set the default value + fwloc = "/boot" + return fwloc + + +def run_userconf_service( + base: str | None, passwd_override: str | None = None +) -> bool: + try: + # reset the TTY device + os.system(f"echo 'reset\\r\\n' > {USERCONF_SERVICE_TTY}") + + time.sleep(1) + # Execute the command on different tty + result = subp.subp( + [ + "openvt", + "-s", + "-f", + "-w", + "-c", + USERCONF_SERVICE_TTY[-1], + "--", + "/usr/lib/userconf-pi/userconf-service", + ], + timeout=(None if not passwd_override else 10), + decode="strict", + ) + + if base: + try: + os.remove(f"{base}/userconf.txt") + except FileNotFoundError: + pass + + if result.stderr: + # Handle failure and restart if needed (Restart=on-failure logic) + LOG.debug(f"Userconf stderr service output: {result.stderr}") + return False + else: + lib_dir = os.path.dirname(MODULE_DEACTIVATION_FILE) + # create deactivation file + os.system( + f"mkdir -p {lib_dir} " "&& touch {MODULE_DEACTIVATION_FILE}" + ) + LOG.debug("Userconf service completed successfully.") + return True + except subprocess.TimeoutExpired: + if base and os.path.exists(f"{base}/failed_userconf.txt"): + LOG.error("Invalid credentials provided for userconf-pi.") + os.remove(f"{base}/failed_userconf.txt") + else: + LOG.error("Userconf service timed out.") + return False + except Exception as e: + LOG.warning("Error running service: %s", e) + if base: + try: + os.remove(f"{base}/userconf.txt") + except FileNotFoundError: + pass + return False + + +def run_service( + passwd_override: str | None = None, user_override: str | None = None +) -> bool: + # Ensure the TTY exists before trying to open it + if not os.path.exists(USERCONF_SERVICE_TTY): + if not passwd_override: + LOG.error("TTY device %s does not exist.", USERCONF_SERVICE_TTY) + return False + else: + LOG.debug("TTY device %s does not exist.", USERCONF_SERVICE_TTY) + + # should never happen and not solvable by the user + assert (passwd_override is None and user_override is None) or ( + passwd_override is not None and user_override is not None + ), ( + "Internal error: User override is required when password " + "override is provided." + ) + + base: str | None = None + if passwd_override: + # write /boot/firmware/userconf.txt + # this will make userconf-service + # run silently with the provided credentials + base = get_fwloc_or_default() + assert base, "Internal error: Failed to get firmware location." + with open(f"{base}/userconf.txt", "w") as f: + f.write(f"{user_override}:{passwd_override}") + LOG.debug("Userconf override file written to %s/userconf.txt", base) + + LOG.debug("Start running userconf-pi service loop...") + while True: + if run_userconf_service(base, passwd_override): + break + # Wait for a moment before retrying + time.sleep(1) + LOG.debug("Userconf-pi service loop: retrying") + LOG.debug("Userconf-pi service loop finished.") + return True + + +def configure_pizwiz( + cfg: Config, + disable: bool, + passwd_override: str | None, + user_override: str | None = None, +) -> None: + LOG.debug( + "Configuring piwiz with disable_piwiz=%s, passwd_override=%s, " + "user_override=%s", + bool_to_str(disable), + bool_to_str(passwd_override is not None), + bool_to_str(user_override is not None), + ) + + if disable: + # execute cancel rename script to ensure + # piwiz isn't started (on desktop) + os.system("/usr/bin/cancel-rename pi") + else: + # execute userconf-pi service + # on desktop this doesn't have any effect + # as piwiz is started by the desktop environment + run_service(passwd_override, user_override) + + # populate users for other cloud-init modules to use + cfg["users"] = cfg.get("users", []) + if user_override and user_override not in cfg["users"]: + cfg["users"].append(user_override) + elif "pi" not in cfg["users"]: + cfg["users"].append("pi") + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + disable_piwiz: bool = False + password_override: str | None = None + user_override: str | None = None + + if os.path.exists(MODULE_DEACTIVATION_FILE) or not os.path.exists( + "/usr/lib/userconf-pi" + ): + LOG.debug( + "Userconf-Pi: deactivation file detected or userconf-pi " + "not installed. Skipping..." + ) + return + + if RPI_USERCONF_KEY in cfg: + # expect it to be a dictionary + userconf = cfg[RPI_USERCONF_KEY] + + # look over miss configuration to + if isinstance(userconf, dict) and "password" in userconf: + password_override = userconf["password"] + # user key is optional with default to pi + user_override = userconf.get("user", "pi") + LOG.debug( + "Userconf override: user=%s, password=", + user_override, + ) + else: + LOG.error("Invalid userconf-pi configuration: %s", userconf) + + if not password_override and DISABLE_PIWIZ_KEY in cfg: + if isinstance(cfg[DISABLE_PIWIZ_KEY], bool): + disable_piwiz = cfg[DISABLE_PIWIZ_KEY] + else: + LOG.error( + "Invalid %s configuration: %s", + str(cfg[DISABLE_PIWIZ_KEY]), + DISABLE_PIWIZ_KEY, + ) + + configure_pizwiz(cfg, disable_piwiz, password_override, user_override) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 3e3bf056d75..7d98832257e 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -23,7 +23,7 @@ meta: MetaSchema = { "id": "cc_ssh_import_id", - "distros": ["alpine", "cos", "debian", "ubuntu"], + "distros": ["alpine", "cos", "debian", "raspberry-pi-os", "ubuntu"], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], } diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index a910c5dbc4b..8a8cc872336 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -50,6 +50,12 @@ "resolv_conf", "rh-subscription", "rh_subscription", + "rpi-connect", + "rpi_connect", + "rpi-interfaces", + "rpi_interfaces", + "rpi-userdata", + "rpi_userdata", "rsyslog", "runcmd", "salt-minion", @@ -2644,6 +2650,98 @@ } } }, + "cc_rpi_connect": { + "type": "object", + "properties": { + "enable_rpi_connect": { + "type": "boolean", + "default": false, + "description": "Install and enable Raspberry Pi Connect. Default: ``false``." + } + } + }, + "cc_rpi_interfaces": { + "type": "object", + "properties": { + "rpi_interfaces": { + "type": "object", + "properties": { + "spi": { + "type": "boolean", + "description": "Enable SPI interface. Default: ``false``.", + "default": false + }, + "i2c": { + "type": "boolean", + "description": "Enable I2C interface. Default: ``false``.", + "default": false + }, + "serial": { + "default": false, + "description": "Enable serial console. Default: ``false``.", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "properties": { + "console": { + "type": "boolean", + "description": "Enable serial console. Default: ``false``.", + "default": false + }, + "hardware": { + "type": "boolean", + "description": "Enable UART hardware. Default: ``false``.", + "default": false + } + } + } + ] + }, + "onewire": { + "type": "boolean", + "description": "Enable 1-Wire interface. Default: ``false``.", + "default": false + }, + "remote_gpio": { + "type": "boolean", + "description": "Enable remote GPIO interface. Default: ``false``.", + "default": false + }, + "ssh": { + "type": "boolean", + "description": "Enable SSH. Default: ``false``.", + "default": false + } + } + } + } + }, + "cc_rpi_userdata": { + "type": "object", + "properties": { + "disable_piwiz": { + "type": "boolean", + "default": false, + "description": "Manually disable the Raspberry Pi first boot wizard. Default: ``false``." + }, + "rpi_userconf": { + "type": "object", + "properties": { + "password": { + "type": "string", + "description": "The password for the default user." + }, + "user": { + "type": "string", + "description": "The username for the default user. Default: ``pi``." + } + } + } + } + }, "cc_rsyslog": { "type": "object", "properties": { @@ -3908,6 +4006,15 @@ { "$ref": "#/$defs/cc_rh_subscription" }, + { + "$ref": "#/$defs/cc_rpi_connect" + }, + { + "$ref": "#/$defs/cc_rpi_interfaces" + }, + { + "$ref": "#/$defs/cc_rpi_userdata" + }, { "$ref": "#/$defs/cc_rsyslog" }, @@ -4004,10 +4111,12 @@ "create_hostname_file": {}, "device_aliases": {}, "disable_ec2_metadata": {}, + "disable_piwiz": {}, "disable_root": {}, "disable_root_opts": {}, "disk_setup": {}, "drivers": {}, + "enable_rpi_connect": {}, "fan": {}, "final_message": {}, "fqdn": {}, @@ -4049,6 +4158,8 @@ "resize_rootfs": {}, "resolv_conf": {}, "rh_subscription": {}, + "rpi_interfaces": {}, + "rpi_userconf": {}, "rsyslog": {}, "runcmd": {}, "salt_minion": {}, diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e9f6d79739a..c11aeb79d5e 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -64,7 +64,7 @@ "alpine": ["alpine"], "aosc": ["aosc"], "arch": ["arch"], - "debian": ["debian", "ubuntu"], + "debian": ["debian", "ubuntu", "raspberry-pi-os"], "freebsd": ["freebsd", "dragonfly"], "gentoo": ["gentoo", "cos"], "netbsd": ["netbsd"], diff --git a/cloudinit/distros/raspberry-pi-os.py b/cloudinit/distros/raspberry-pi-os.py new file mode 100644 index 00000000000..c0bfa097e1a --- /dev/null +++ b/cloudinit/distros/raspberry-pi-os.py @@ -0,0 +1,45 @@ +# Copyright (C) 2024 Raspberry Pi Ltd. All rights reserved. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import subp +from cloudinit.distros import debian + + +class Distro(debian.Distro): + def __init__(self, name, cfg, paths): + super().__init__(name, cfg, paths) + + def set_keymap(self, layout: str, model: str, variant: str, options: str): + """Currently Raspberry Pi OS sys-mods only supports + setting the layout""" + + subp.subp( + [ + "/usr/lib/raspberrypi-sys-mods/imager_custom", + "set_keymap", + layout, + ] + ) + + def apply_locale(self, locale, out_fn=None, keyname="LANG"): + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + f"{locale}", + ] + ) + except subp.ProcessExecutionError: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + f"{locale}.UTF-8", + ] + ) diff --git a/cloudinit/util.py b/cloudinit/util.py index bfcc9c8edba..348cd863f12 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -550,6 +550,8 @@ def get_linux_distro(): os_release_rhel = False if os.path.exists("/etc/os-release"): os_release = load_shell_content(load_text_file("/etc/os-release")) + if os.path.exists("/etc/rpi-issue"): + os_release["ID"] = "raspberry-pi-os" if not os_release: os_release_rhel = True os_release = _parse_redhat_release() @@ -625,6 +627,7 @@ def _get_variant(info): "opencloudos", "openmandriva", "photon", + "raspberry-pi-os", "rhel", "rocky", "suse", diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 6c891a35e1f..d3e8ff8f655 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -10,6 +10,7 @@ "mariner": "MarinerOS", "rhel": "Cloud User", "netbsd": "NetBSD", "openbsd": "openBSD", "openmandriva": "OpenMandriva admin", "photon": "PhotonOS", + "raspberry-pi-os": "Raspberry Pi OS", "ubuntu": "Ubuntu", "unknown": "Ubuntu"}) %} {% set groups = ({"alpine": "adm, wheel", "aosc": "wheel", "arch": "wheel, users", "azurelinux": "wheel", @@ -17,6 +18,7 @@ "gentoo": "users, wheel", "mariner": "wheel", "photon": "wheel", "openmandriva": "wheel, users, systemd-journal", + "raspberry-pi-os": "adm, dialout, cdrom, audio, users, sudo, video, games, plugdev, input, gpio, spi, i2c, netdev, render", "suse": "cdrom, users", "ubuntu": "adm, cdrom, dip, lxd, sudo", "unknown": "adm, cdrom, dip, lxd, sudo"}) %} @@ -24,7 +26,8 @@ "freebsd": "/bin/tcsh", "netbsd": "/bin/sh", "openbsd": "/bin/ksh"}) %} {% set usernames = ({"amazon": "ec2-user", "centos": "cloud-user", - "openmandriva": "omv", "rhel": "cloud-user", + "openmandriva": "omv", "raspberry-pi-os": "pi", + "rhel": "cloud-user", "unknown": "ubuntu"}) %} {% if is_bsd %} syslog_fix_perms: root:wheel @@ -134,9 +137,15 @@ cloud_init_modules: - ca_certs {% endif %} - rsyslog +{% if variant == "raspberry-pi-os" %} + - rpi_userdata +{% endif %} - users_groups - ssh - set_passwords +{% if variant == "raspberry-pi-os" %} + - rpi_interfaces +{% endif %} # The modules that run in the 'config' stage cloud_config_modules: @@ -156,6 +165,9 @@ cloud_config_modules: {% endif %} - locale {% endif %} +{% if variant == "raspberry-pi-os" %} + - rpi_interfaces +{% endif %} {% if variant == "alpine" %} - apk_configure {% elif variant in ["debian", "ubuntu", "unknown"] %} @@ -204,6 +216,9 @@ cloud_final_modules: - chef {% endif %} - ansible +{% if variant == "raspberry-pi-os" %} + - rpi_connect +{% endif %} {% if variant not in ["azurelinux"] %} - mcollective - salt_minion @@ -229,8 +244,8 @@ system_info: # This will affect which distro class gets used {% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "freebsd", "gentoo", "mariner", "netbsd", "openbsd", - "OpenCloudOS", "openeuler", "openmandriva", "photon", "suse", - "TencentOS", "ubuntu"] or is_rhel %} + "OpenCloudOS", "openeuler", "openmandriva", "photon", + "raspberry-pi-os", "suse", "TencentOS", "ubuntu"] or is_rhel %} distro: {{ variant }} {% elif variant == "dragonfly" %} distro: dragonflybsd @@ -247,8 +262,8 @@ system_info: {% endif %} {% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", - "openmandriva", "photon", "suse", "TencentOS", "ubuntu", - "unknown"] + "openmandriva", "photon", "raspberry-pi-os", + "suse", "TencentOS", "ubuntu", "unknown"] or is_bsd or is_rhel %} lock_passwd: True {% endif %} @@ -307,6 +322,11 @@ system_info: {% elif variant == "openmandriva" %} network: renderers: ['network-manager', 'networkd'] +{% elif variant == "raspberry-pi-os" %} + network: + dhcp_client_priority: [dhclient, dhcpcd, udhcpc] + renderers: ['netplan', 'network-manager', 'networkd', 'eni'] + activators: ['netplan', 'network-manager', 'networkd', 'eni'] {% elif variant in ["ubuntu", "unknown"] %} {# SRU_BLOCKER: do not ship network renderers on Xenial, Bionic or Eoan #} network: @@ -346,6 +366,19 @@ system_info: failsafe: primary: https://deb.debian.org/debian security: https://deb.debian.org/debian-security +{% elif variant == "raspberry-pi-os" %} + package_mirrors: + - arches: [arm64] + failsafe: + primary: + - https://deb.debian.org/debian + - http://archive.raspberrypi.com/debian/ + security: https://deb.debian.org/debian-security + - arches: [armhf] + failsafe: + primary: + - http://raspbian.raspberrypi.com/raspbian/ + - http://archive.raspberrypi.com/debian/ {% elif variant in ["ubuntu", "unknown"] %} package_mirrors: - arches: [i386, amd64] @@ -373,7 +406,7 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports {% endif %} -{% if variant in ["debian", "ubuntu", "unknown"] %} +{% if variant in ["debian", "raspberry-pi-os", "ubuntu", "unknown"] %} ssh_svcname: ssh {% elif variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", diff --git a/doc/module-docs/cc_rpi_connect/data.yaml b/doc/module-docs/cc_rpi_connect/data.yaml new file mode 100644 index 00000000000..c738697c4f3 --- /dev/null +++ b/doc/module-docs/cc_rpi_connect/data.yaml @@ -0,0 +1,13 @@ +cc_rpi_connect: + description: | + This module handles Raspberry Pi Connect installation and enablement. + It will install the Raspberry Pi Connect service and enable it to run on boot. + + Supported operating systems: + - Raspberry Pi OS (bookworm and later) + examples: + - comment: > + This example will enable the Raspberry Pi Connect service. + file: cc_rpi_connect/example1.yaml + name: Raspberry Pi Connect + title: Install and enable Raspberry Pi Connect service diff --git a/doc/module-docs/cc_rpi_connect/example1.yaml b/doc/module-docs/cc_rpi_connect/example1.yaml new file mode 100644 index 00000000000..61b46e62b13 --- /dev/null +++ b/doc/module-docs/cc_rpi_connect/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +enable_rpi_connect: true diff --git a/doc/module-docs/cc_rpi_interfaces/data.yaml b/doc/module-docs/cc_rpi_interfaces/data.yaml new file mode 100644 index 00000000000..168ee408c81 --- /dev/null +++ b/doc/module-docs/cc_rpi_interfaces/data.yaml @@ -0,0 +1,19 @@ +cc_rpi_interfaces: + description: | + This module handles ARM interface configuration for Raspberry Pi. + I also has an option to enable the ssh service which is disabled by default. + examples: + - comment: > + This example will enable the SPI and I2C interfaces on Raspberry Pi. + file: cc_rpi_interfaces/example1.yaml + - comment: > + This example will enable the serial interface on Raspberry Pi. + file: cc_rpi_interfaces/example2.yaml + - comment: > + This example will enable the serial interface on Raspberry Pi 5 and disable the UART hardware while enabling the console. + file: cc_rpi_interfaces/example3.yaml + - comment: > + This example will enable ssh and the UART hardware without binding it to the console. + file: cc_rpi_interfaces/example4.yaml + name: Raspberry Pi Interfaces + title: Configure Raspberry Pi ARM interfaces diff --git a/doc/module-docs/cc_rpi_interfaces/example1.yaml b/doc/module-docs/cc_rpi_interfaces/example1.yaml new file mode 100644 index 00000000000..68a0268851a --- /dev/null +++ b/doc/module-docs/cc_rpi_interfaces/example1.yaml @@ -0,0 +1,4 @@ +#cloud-config +rpi_interfaces: + spi: true + i2c: true diff --git a/doc/module-docs/cc_rpi_interfaces/example2.yaml b/doc/module-docs/cc_rpi_interfaces/example2.yaml new file mode 100644 index 00000000000..eeae5fd71c4 --- /dev/null +++ b/doc/module-docs/cc_rpi_interfaces/example2.yaml @@ -0,0 +1,3 @@ +#cloud-config +rpi_interfaces: + serial: true diff --git a/doc/module-docs/cc_rpi_interfaces/example3.yaml b/doc/module-docs/cc_rpi_interfaces/example3.yaml new file mode 100644 index 00000000000..b7eb744c243 --- /dev/null +++ b/doc/module-docs/cc_rpi_interfaces/example3.yaml @@ -0,0 +1,6 @@ +#cloud-config +rpi_interfaces: + serial: + # Pi 5 only | disabling hardware while enabling console + console: true + hardware: false diff --git a/doc/module-docs/cc_rpi_interfaces/example4.yaml b/doc/module-docs/cc_rpi_interfaces/example4.yaml new file mode 100644 index 00000000000..4822fd4ae70 --- /dev/null +++ b/doc/module-docs/cc_rpi_interfaces/example4.yaml @@ -0,0 +1,8 @@ +#cloud-config +rpi_interfaces: + ssh: true + # works on all Pi models + # only enables the UART hardware without binding it to the console + serial: + console: false + hardware: true diff --git a/doc/module-docs/cc_rpi_userdata/data.yaml b/doc/module-docs/cc_rpi_userdata/data.yaml new file mode 100644 index 00000000000..04a9ebc3ee1 --- /dev/null +++ b/doc/module-docs/cc_rpi_userdata/data.yaml @@ -0,0 +1,26 @@ +cc_rpi_userdata: + description: | + A cloud-init module that is intented to run in the final stage of the cloud-init process. + + It will take care of configuring and triggering the setup wizards on Raspberry Pi OS. + + On desktop images, it will configure the setup wizard (piwiz) or + disable it if requested. + + On lite images, it will configure/disable the userconf-pi package + and trigger the userconf-pi setup wizard. + + Supported operating systems: + - Raspberry Pi OS (bookworm and later) + examples: + - comment: > + This example will manually disable the setup wizard. + file: cc_rpi_userdata/example1.yaml + - comment: > + This example will set a password for the default pi user. (disables the setup wizard) + file: cc_rpi_userdata/example2.yaml + - comment: > + This example will trigger a rename of the default pi user and set a password. (disables the setup wizard) + file: cc_rpi_userdata/example3.yaml + name: Raspberry Pi OS Userdata + title: Configure and trigger Raspberry Pi OS setup wizards diff --git a/doc/module-docs/cc_rpi_userdata/example1.yaml b/doc/module-docs/cc_rpi_userdata/example1.yaml new file mode 100644 index 00000000000..e6b284d7b73 --- /dev/null +++ b/doc/module-docs/cc_rpi_userdata/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +disable_piwiz: true diff --git a/doc/module-docs/cc_rpi_userdata/example2.yaml b/doc/module-docs/cc_rpi_userdata/example2.yaml new file mode 100644 index 00000000000..93d9edb8348 --- /dev/null +++ b/doc/module-docs/cc_rpi_userdata/example2.yaml @@ -0,0 +1,3 @@ +#cloud-config +rpi_userconf: + password: "my_super_secret_password_hash" diff --git a/doc/module-docs/cc_rpi_userdata/example3.yaml b/doc/module-docs/cc_rpi_userdata/example3.yaml new file mode 100644 index 00000000000..3b98695de3c --- /dev/null +++ b/doc/module-docs/cc_rpi_userdata/example3.yaml @@ -0,0 +1,4 @@ +#cloud-config +rpi_userconf: + user: "my_new_user_name" + password: "my_super_secret_password_hash" diff --git a/doc/rtd/reference/availability.rst b/doc/rtd/reference/availability.rst index c4762208005..5f6b983c55d 100644 --- a/doc/rtd/reference/availability.rst +++ b/doc/rtd/reference/availability.rst @@ -39,6 +39,7 @@ NetBSD, OpenBSD and DragonFlyBSD: - OpenCloudOS - OpenMandriva - Photon OS +- Raspberry Pi OS - Red Hat Enterprise Linux (RHEL) - Rocky Linux - SLES/openSUSE diff --git a/doc/rtd/reference/modules.rst b/doc/rtd/reference/modules.rst index c5cdef1620a..59f2ddd8b3c 100644 --- a/doc/rtd/reference/modules.rst +++ b/doc/rtd/reference/modules.rst @@ -80,6 +80,12 @@ Modules :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_rh_subscription/data.yaml :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_rpi_connect/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_rpi_interfaces/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_rpi_userdata/data.yaml + :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_rsyslog/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_runcmd/data.yaml diff --git a/doc/rtd/reference/network-config.rst b/doc/rtd/reference/network-config.rst index 46d4d977e76..5bd99c9a4d7 100644 --- a/doc/rtd/reference/network-config.rst +++ b/doc/rtd/reference/network-config.rst @@ -182,13 +182,13 @@ supports a wide range of networking setups. Configuration is typically stored in :file:`/etc/NetworkManager`. It is the default for a number of Linux distributions; notably Fedora, -CentOS/RHEL, and their derivatives. +CentOS/RHEL, Raspberry Pi OS, and their derivatives. ENI --- :file:`/etc/network/interfaces` or ``ENI`` is supported by the ``ifupdown`` -package found in Alpine Linux, Debian and Ubuntu. +package found in Alpine Linux, Debian, Raspberry Pi OS and Ubuntu. Netplan ------- @@ -270,7 +270,7 @@ Example output: .. code-block:: usage: /usr/bin/cloud-init devel net-convert [-h] -p PATH -k {eni,network_data.json,yaml,azure-imds,vmware-imc} -d PATH -D - {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler} + {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler,raspberry-pi-os} [-m name,mac] [--debug] -O {eni,netplan,networkd,sysconfig,network-manager} options: @@ -281,7 +281,7 @@ Example output: The format of the given network config -d PATH, --directory PATH directory to place output in - -D {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openeuler}, --distro {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler} + -D {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openeuler}, --distro {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler,raspberry-pi-os} -m name,mac, --mac name,mac interface name to mac mapping --debug enable debug logging to stderr. diff --git a/pyproject.toml b/pyproject.toml index faf09c234f7..0c15283edbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,6 +132,14 @@ module = [ "tests.unittests.config.test_cc_resizefs", "tests.unittests.config.test_cc_resolv_conf", "tests.unittests.config.test_cc_rh_subscription", + "tests.unittests.config.test_cc_rpi_connect", + "tests.unittests.config.test_cc_rpi_interfaces", + "tests.unittests.config.test_cc_rpi_userdata", + "tests.unittests.config.test_cc_rsyslog", + "tests.unittests.config.test_cc_runcmd", + "tests.unittests.config.test_cc_snap", + "tests.unittests.config.test_cc_ssh", + "tests.unittests.config.test_cc_timezone", "tests.unittests.config.test_cc_ubuntu_autoinstall", "tests.unittests.config.test_cc_update_etc_hosts", "tests.unittests.config.test_cc_users_groups", diff --git a/systemd/cloud-init-local.service.tmpl b/systemd/cloud-init-local.service.tmpl index 3bbbc7f0bbe..26a6aee1d05 100644 --- a/systemd/cloud-init-local.service.tmpl +++ b/systemd/cloud-init-local.service.tmpl @@ -2,7 +2,7 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Local Stage (pre-network) -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "rhel"] %} +{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=network-pre.target @@ -13,7 +13,7 @@ Before=shutdown.target {% if variant in ["almalinux", "cloudlinux", "rhel"] %} Before=firewalld.target {% endif %} -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} Before=sysinit.target {% endif %} Conflicts=shutdown.target diff --git a/systemd/cloud-init-main.service.tmpl b/systemd/cloud-init-main.service.tmpl index b80f324fe7a..751259df990 100644 --- a/systemd/cloud-init-main.service.tmpl +++ b/systemd/cloud-init-main.service.tmpl @@ -8,7 +8,8 @@ # https://www.freedesktop.org/software/systemd/man/latest/systemd-remount-fs.service.html [Unit] Description=Cloud-init: Single Process -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "rhel"] %} +Wants=network-pre.target +{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} {% if variant in ["almalinux", "cloudlinux", "rhel"] %} diff --git a/systemd/cloud-init-network.service.tmpl b/systemd/cloud-init-network.service.tmpl index bdc7c8f8369..d005f8fed0f 100644 --- a/systemd/cloud-init-network.service.tmpl +++ b/systemd/cloud-init-network.service.tmpl @@ -12,8 +12,12 @@ After=cloud-init-local.service {% if variant not in ["ubuntu"] %} After=systemd-networkd-wait-online.service {% endif %} -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} After=networking.service +{% if variant == "raspberry-pi-os" %} +Wants=NetworkManager-wait-online.service +After=NetworkManager-wait-online.service +{% endif %} {% endif %} {% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", @@ -28,11 +32,17 @@ After=wicked.service After=dbus.service {% endif %} Before=network-online.target +{% if variant == "raspberry-pi-os" %} +Before=avahi-daemon.service +{% endif %} Before=sshd-keygen.service Before=sshd.service Before=systemd-user-sessions.service -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} +{% if variant not in ["raspberry-pi-os"] %} +{# stops cloud-init-network from starting on rpios #} Before=sysinit.target +{% endif %} Before=shutdown.target Conflicts=shutdown.target {% endif %} diff --git a/tests/unittests/config/test_cc_rpi_connect.py b/tests/unittests/config/test_cc_rpi_connect.py new file mode 100644 index 00000000000..19284e49864 --- /dev/null +++ b/tests/unittests/config/test_cc_rpi_connect.py @@ -0,0 +1,76 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config.cc_rpi_connect import ENABLE_RPI_CONNECT_KEY +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema +import pytest + + +""" +def is_notPi() -> bool: + \"""Most tests are Raspberry Pi OS only.\""" + return not os.path.exists("/etc/rpi-issue") + +@mock.patch('cloudinit.subp.subp') +class TestCCRPiConnect(CiTestCase): + \"""Tests work in progress. Just partially implemented to + show the idea.\""" + + @mock.patch('cloudinit.subp.subp') + def test_configure_rpi_connect_enabled(self, mock_subp): + if is_notPi(): + return + config = {ENABLE_RPI_CONNECT_KEY: True} + handle("cc_rpi_connect", config, mock.Mock(), []) + mock_subp.assert_called_with( + ['/usr/bin/raspi-config', 'do_rpi_connect', '0']) + + @mock.patch('cloudinit.subp.subp') + def test_configure_rpi_connect_disabled(self, mock_subp): + if is_notPi(): + return + config = {ENABLE_RPI_CONNECT_KEY: False} + handle("cc_rpi_connect", config, mock.Mock(), []) + mock_subp.assert_called_with( + ['/usr/bin/raspi-config', 'do_rpi_connect', '1']) + + @mock.patch('os.path.exists') + def test_rpi_connect_installed(self, mock_path_exists): + if is_notPi(): + return + # Simulate rpi-connect is installed + mock_path_exists.return_value = True + assert mock_path_exists('/usr/bin/rpi-connect') + + @mock.patch('os.path.exists') + def test_rpi_connect_not_installed(self, mock_path_exists): + if is_notPi(): + return + # Simulate rpi-connect is not installed + mock_path_exists.return_value = False + assert not mock_path_exists('/usr/bin/rpi-connect') +""" + + +@skipUnlessJsonSchema() +class TestCCRPiConnectSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + ({ENABLE_RPI_CONNECT_KEY: True}, None), + ( + {ENABLE_RPI_CONNECT_KEY: "true"}, + "'true' is not of type 'boolean'", + ), + ], + ) + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_rpi_interfaces.py b/tests/unittests/config/test_cc_rpi_interfaces.py new file mode 100644 index 00000000000..c9d55db2de6 --- /dev/null +++ b/tests/unittests/config/test_cc_rpi_interfaces.py @@ -0,0 +1,124 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config.cc_rpi_interfaces import RPI_INTERFACES_KEY +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema +import pytest + + +""" +def is_notPi() -> bool: + \"""Most tests are Raspberry Pi OS only.\""" + return not os.path.exists("/etc/rpi-issue") + +@mock.patch('cloudinit.subp.subp') +class TestCCRPiInterfaces(CiTestCase): + \"""Tests work in progress. Just partially implemented to + show the idea.\""" + + @mock.patch('cloudinit.subp.subp') + def test_configure_spi_interface(self, mock_subp): + if is_notPi(): + return + config = { + RPI_INTERFACES_KEY: { + "spi": True + } + } + handle("cc_rpi_interfaces", config, mock.Mock(), []) + mock_subp.assert_called_with([ + '/usr/bin/raspi-config', + 'nonint', + SUPPORTED_INTERFACES["spi"], + '0']) + + @mock.patch('cloudinit.subp.subp') + def test_configure_serial_interface_as_dict(self, mock_subp): + if is_notPi(): + return + config = { + RPI_INTERFACES_KEY: { + "serial": { + "console": True, + "hardware": False + } + } + } + handle("cc_rpi_interfaces", config, mock.Mock(), []) + mock_subp.assert_any_call([ + '/usr/bin/raspi-config', 'nonint', 'do_serial_cons', '0']) + + @mock.patch('cloudinit.subp.subp') + def test_configure_invalid_interface(self, mock_subp): + if is_notPi(): + return + config = { + RPI_INTERFACES_KEY: { + "unknown_interface": True + } + } + handle("cc_rpi_interfaces", config, mock.Mock(), []) + mock_subp.assert_not_called() + + @mock.patch('os.path.exists') + @mock.patch('cloudinit.subp.subp') + def test_get_enabled_interfaces(self, mock_subp, mock_path_exists): + if is_notPi(): + return + # Simulate all interfaces enabled (spi, i2c, etc.) + mock_subp.side_effect = [("0", ""), ("0", ""), ("0", ""), ("0", "")] + config = { + RPI_INTERFACES_KEY: { + "spi": True, + "i2c": True, + "onewire": True, + "remote_gpio": True + } + } + handle("cc_rpi_interfaces", config, mock.Mock(), []) + # Assert all interface enabling commands were called + mock_subp.assert_any_call(['/usr/bin/raspi-config', + 'nonint', 'do_spi', '0']) + mock_subp.assert_any_call(['/usr/bin/raspi-config', + 'nonint', 'do_i2c', '0']) + mock_subp.assert_any_call(['/usr/bin/raspi-config', + 'nonint', 'do_onewire', '0']) + mock_subp.assert_any_call(['/usr/bin/raspi-config', + 'nonint', 'do_rgpio', '0']) +""" + + +@skipUnlessJsonSchema() +class TestCCRPiInterfacesSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + ({RPI_INTERFACES_KEY: {"spi": True, "i2c": False}}, None), + ( + {RPI_INTERFACES_KEY: {"spi": "true"}}, + "'true' is not of type 'boolean'", + ), + ( + { + RPI_INTERFACES_KEY: { + "serial": {"console": True, "hardware": False} + } + }, + None, + ), + ( + {RPI_INTERFACES_KEY: {"serial": {"console": 123}}}, + "123 is not of type 'boolean'", + ), + ], + ) + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_rpi_userdata.py b/tests/unittests/config/test_cc_rpi_userdata.py new file mode 100644 index 00000000000..75d78b05018 --- /dev/null +++ b/tests/unittests/config/test_cc_rpi_userdata.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config.cc_rpi_userdata import ( + DISABLE_PIWIZ_KEY, + RPI_USERCONF_KEY, +) +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +import pytest +from tests.unittests.helpers import skipUnlessJsonSchema + +""" +def is_notPi() -> bool: + \"""Most tests are Raspberry Pi OS only.\""" + return not os.path.exists("/etc/rpi-issue") + +@mock.patch('cloudinit.subp.subp') +class TestCCRPiUserdata(CiTestCase): + \"""Tests work in progress. Just partially implemented + to show the idea.\""" + + @mock.patch('subprocess.run') + def test_userconf_service_runs(self, mock_run): + if is_notPi(): + return + mock_run.return_value.returncode = 0 + result = run_service('hashedpassword', 'newuser') + assert result is True + + @mock.patch('subprocess.run') + def test_userconf_service_fails(self, mock_run): + if is_notPi(): + return + mock_run.return_value.returncode = 1 + result = run_service('hashedpassword', 'newuser') + assert result is False + + @mock.patch('os.path.exists') + def test_check_piwiz_disabled(self, mock_path_exists): + if is_notPi(): + return + mock_path_exists.side_effect = [False, False, False] + assert not mock_path_exists('/var/lib/userconf-pi/autologin') + assert not mock_path_exists('/etc/ssh/sshd_config.d/rename_user.conf') + assert not mock_path_exists('/etc/xdg/autostart/piwiz.desktop') + + @mock.patch('os.listdir') + def test_check_default_user_renamed(self, mock_listdir): + if is_notPi(): + return + mock_listdir.return_value = ['newuser'] + assert 'newuser' in os.listdir('/home') + + @mock.patch('os.listdir') + def test_default_user_still_exists(self, mock_listdir): + if is_notPi(): + return + mock_listdir.return_value = ['pi'] + assert 'pi' in os.listdir('/home') +""" + + +@skipUnlessJsonSchema() +class TestCCRPiUserdataSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + ({DISABLE_PIWIZ_KEY: True}, None), + ( + { + RPI_USERCONF_KEY: { + "password": "hashedpassword", + "user": "newuser", + } + }, + None, + ), + ({DISABLE_PIWIZ_KEY: "true"}, "'true' is not of type 'boolean'"), + ( + {RPI_USERCONF_KEY: {"password": 12345}}, + "12345 is not of type 'string'", + ), + ], + ) + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 3da6d925c02..8a64c4917b1 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -257,6 +257,9 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_resizefs"}, {"$ref": "#/$defs/cc_resolv_conf"}, {"$ref": "#/$defs/cc_rh_subscription"}, + {"$ref": "#/$defs/cc_rpi_connect"}, + {"$ref": "#/$defs/cc_rpi_interfaces"}, + {"$ref": "#/$defs/cc_rpi_userdata"}, {"$ref": "#/$defs/cc_rsyslog"}, {"$ref": "#/$defs/cc_runcmd"}, {"$ref": "#/$defs/cc_salt_minion"}, diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 0ed9464821d..dde51a69904 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -22,6 +22,7 @@ "netbsd", "openbsd", "photon", + "raspberry-pi-os", "rhel", "suse", "ubuntu", @@ -93,6 +94,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): "amazon": "ec2-user", "rhel": "cloud-user", "centos": "cloud-user", + "raspberry-pi-os": "pi", "unknown": "ubuntu", } default_user = system_cfg["system_info"]["default_user"]["name"] @@ -105,6 +107,10 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): ("netbsd", ["netbsd"]), ("openbsd", ["openbsd"]), ("ubuntu", ["netplan", "eni", "sysconfig"]), + ( + "raspberry-pi-os", + ["netplan", "network-manager", "networkd", "eni"] + ) ), ) def test_variant_sets_network_renderer_priority_in_cloud_cfg( diff --git a/tools/render-template b/tools/render-template index 78beeecb2cf..4b5efcf347b 100755 --- a/tools/render-template +++ b/tools/render-template @@ -34,6 +34,7 @@ def main(): "OpenCloudOS", "openmandriva", "photon", + "raspberry-pi-os", "rhel", "suse", "rocky", From e0b6ecd4cc7d5f6ee4ecafbe48e0f804d17cdc11 Mon Sep 17 00:00:00 2001 From: paulober Date: Thu, 17 Oct 2024 17:26:33 +0100 Subject: [PATCH 02/20] Remove legacy dhcp Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- config/cloud.cfg.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index d3e8ff8f655..220ec03dda1 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -324,7 +324,7 @@ system_info: renderers: ['network-manager', 'networkd'] {% elif variant == "raspberry-pi-os" %} network: - dhcp_client_priority: [dhclient, dhcpcd, udhcpc] + dhcp_client_priority: [dhclient] renderers: ['netplan', 'network-manager', 'networkd', 'eni'] activators: ['netplan', 'network-manager', 'networkd', 'eni'] {% elif variant in ["ubuntu", "unknown"] %} From a2149f66934c4966584bf9e41dae0dfe18a3ecd1 Mon Sep 17 00:00:00 2001 From: paulober Date: Thu, 17 Oct 2024 17:41:02 +0100 Subject: [PATCH 03/20] Add only available NTP client Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- config/cloud.cfg.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 220ec03dda1..c4271ac869b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -346,6 +346,8 @@ system_info: {% if variant in ["debian", "ubuntu", "unknown"] %} # Automatically discover the best ntp_client ntp_client: auto +{% elif variant == "raspberry-pi-os" %} + ntp_client: 'systemd-timesyncd' {% endif %} {% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", From 69712bf430f8c390f744723186b05b736486b144 Mon Sep 17 00:00:00 2001 From: paulober Date: Fri, 18 Oct 2024 11:20:00 +0100 Subject: [PATCH 04/20] Update raspberry-pi-os config Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- config/cloud.cfg.tmpl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index c4271ac869b..52e3a979bc6 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -324,9 +324,8 @@ system_info: renderers: ['network-manager', 'networkd'] {% elif variant == "raspberry-pi-os" %} network: - dhcp_client_priority: [dhclient] - renderers: ['netplan', 'network-manager', 'networkd', 'eni'] - activators: ['netplan', 'network-manager', 'networkd', 'eni'] + renderers: ['netplan', 'network-manager'] + activators: ['netplan', 'network-manager'] {% elif variant in ["ubuntu", "unknown"] %} {# SRU_BLOCKER: do not ship network renderers on Xenial, Bionic or Eoan #} network: From 2acce87171f521c73b1920a69a9d6afdb4546f3c Mon Sep 17 00:00:00 2001 From: paulober Date: Fri, 18 Oct 2024 11:42:14 +0100 Subject: [PATCH 05/20] Add cc_netplan_nm_patch module Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_netplan_nm_patch.py | 129 ++++++++++++++++++ .../schemas/schema-cloud-config-v1.json | 9 ++ doc/module-docs/cc_netplan_nm_patch/data.yaml | 13 ++ doc/rtd/reference/modules.rst | 2 + .../config/test_cc_netplan_nm_patch.py | 9 ++ tests/unittests/config/test_schema.py | 1 + tests/unittests/test_render_template.py | 2 +- 7 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/cc_netplan_nm_patch.py create mode 100644 doc/module-docs/cc_netplan_nm_patch/data.yaml create mode 100644 tests/unittests/config/test_cc_netplan_nm_patch.py diff --git a/cloudinit/config/cc_netplan_nm_patch.py b/cloudinit/config/cc_netplan_nm_patch.py new file mode 100644 index 00000000000..49d6eb698d2 --- /dev/null +++ b/cloudinit/config/cc_netplan_nm_patch.py @@ -0,0 +1,129 @@ +# Copyright (C) 2024, Raspberry Pi Ltd. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import subp +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema +from cloudinit.distros import ALL_DISTROS +from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE +from cloudinit.settings import PER_INSTANCE +import logging +import os +import re + +LOG = logging.getLogger(__name__) + +meta: MetaSchema = { + "id": "cc_netplan_nm_patch", + "distros": [ALL_DISTROS], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": [], +} + + +def exec_cmd(command: str) -> str | None: + try: + result = subp.subp(command) + if result.stdout is not None: + return result.stdout + except subp.ProcessExecutionError as e: + LOG.error("Failed to execute command: %s", e) + return None + LOG.debug("Command has no stdout: %s", command) + return None + + +def get_netplan_generated_configs() -> list[str]: + """Get the UUIDs of all connections starting with 'netplan-'.""" + output = exec_cmd(["nmcli", "connection", "show"]) + if output is None: + return [] + + netplan_conns = [] + for line in output.splitlines(): + if line.startswith("netplan-"): + parts = line.split() + if len(parts) > 1: + # name = parts[0] + uuid = parts[1] + netplan_conns.append(uuid) + return netplan_conns + + +def get_connection_object_path(uuid: str) -> str | None: + """Get the D-Bus object path for a connection by UUID.""" + output = exec_cmd( + [ + "busctl", + "call", + "org.freedesktop.NetworkManager", + "/org/freedesktop/NetworkManager/Settings", + "org.freedesktop.NetworkManager.Settings", + "GetConnectionByUuid", + "s", + uuid, + ] + ) + + path_match = ( + re.search( + r'o\s+"(/org/freedesktop/NetworkManager/Settings/\d+)"', output + ) + if output + else None + ) + if path_match: + return path_match.group(1) + else: + LOG.error("Failed to find object path for connection: %s", uuid) + return None + + +def save_connection(obj_path: str) -> None: + """Call the Save method on the D-Bus obj path for a connection.""" + result = exec_cmd( + [ + "busctl", + "call", + "org.freedesktop.NetworkManager", + obj_path, + "org.freedesktop.NetworkManager.Settings.Connection", + "Save", + ] + ) + + if result is None: + LOG.error("Failed to save connection: %s", obj_path) + else: + LOG.debug("Saved connection: %s", obj_path) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + LOG.debug("Applying netplan patch") + + # remove cloud-init file after NetworkManager has generated + # replacement netplan configurations to avoid conflicts in the + # future + + try: + np_conns = get_netplan_generated_configs() + if not np_conns: + LOG.debug("No netplan connections found") + return + + for conn_uuid in np_conns: + obj_path = get_connection_object_path(conn_uuid) + if obj_path is None: + continue + save_connection(obj_path) + + os.remove(CLOUDINIT_NETPLAN_FILE) + LOG.debug("Netplan cfg has been patched: %s", CLOUDINIT_NETPLAN_FILE) + except subp.ProcessExecutionError as e: + LOG.error("Failed to patch netplan cfg: %s", e) + except Exception as e: + LOG.error("Failed to patch netplan cfg: %s", e) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index 8a8cc872336..0f345e74e7a 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -35,6 +35,8 @@ "lxd", "mcollective", "mounts", + "netplan-nm-patch", + "netplan_nm_patch", "ntp", "package-update-upgrade-install", "package_update_upgrade_install", @@ -2141,6 +2143,10 @@ } } }, + "cc_netplan_nm_patch": { + "type": "object", + "properties": {} + }, "cc_ntp": { "type": "object", "properties": { @@ -3982,6 +3988,9 @@ { "$ref": "#/$defs/cc_mounts" }, + { + "$ref": "#/$defs/cc_netplan_nm_patch" + }, { "$ref": "#/$defs/cc_ntp" }, diff --git a/doc/module-docs/cc_netplan_nm_patch/data.yaml b/doc/module-docs/cc_netplan_nm_patch/data.yaml new file mode 100644 index 00000000000..f07c41688d6 --- /dev/null +++ b/doc/module-docs/cc_netplan_nm_patch/data.yaml @@ -0,0 +1,13 @@ +cc_netplan_nm_patch: + description: | + This module resolves the issue where if you edit a netplan generated network configuration + in NetworkManager, the changes are saved to netplan as new configuration files. + This will cause netplan overwrite the NetworkManager generated netplan configuration on the next boot. + + This module will give NetworkManager a hint to generated netplan configuration files + and then removes the cloud-init generated netplan configuration file. + + This module expects that NetworkManager has the netplan integration patch applied. + examples: + name: Netplan NetworkManager Patch + title: Patch NetworkManager netplan interop issue diff --git a/doc/rtd/reference/modules.rst b/doc/rtd/reference/modules.rst index 59f2ddd8b3c..aec67115598 100644 --- a/doc/rtd/reference/modules.rst +++ b/doc/rtd/reference/modules.rst @@ -64,6 +64,8 @@ Modules :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_mounts/data.yaml :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_netplan_nm_patch/data.yaml + :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_ntp/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_package_update_upgrade_install/data.yaml diff --git a/tests/unittests/config/test_cc_netplan_nm_patch.py b/tests/unittests/config/test_cc_netplan_nm_patch.py new file mode 100644 index 00000000000..6f76cba7446 --- /dev/null +++ b/tests/unittests/config/test_cc_netplan_nm_patch.py @@ -0,0 +1,9 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from tests.unittests.helpers import skipUnlessJsonSchema + + +@skipUnlessJsonSchema() +class TestCCNetplanNmPatch: + def test_schema_validation(self): + pass diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 8a64c4917b1..36c47e8f17e 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -249,6 +249,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_lxd"}, {"$ref": "#/$defs/cc_mcollective"}, {"$ref": "#/$defs/cc_mounts"}, + {"$ref": "#/$defs/cc_netplan_nm_patch"}, {"$ref": "#/$defs/cc_ntp"}, {"$ref": "#/$defs/cc_package_update_upgrade_install"}, {"$ref": "#/$defs/cc_phone_home"}, diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index dde51a69904..9cd124506a3 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -109,7 +109,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): ("ubuntu", ["netplan", "eni", "sysconfig"]), ( "raspberry-pi-os", - ["netplan", "network-manager", "networkd", "eni"] + ["netplan", "network-manager"] ) ), ) From 18b3d10b56904f18cbe3c39999f8203bc9f4cdb9 Mon Sep 17 00:00:00 2001 From: paulober Date: Tue, 10 Dec 2024 12:58:30 +0100 Subject: [PATCH 06/20] Fix some lint errors Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_netplan_nm_patch.py | 2 +- cloudinit/config/cc_rpi_connect.py | 2 +- cloudinit/config/cc_rpi_userdata.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_netplan_nm_patch.py b/cloudinit/config/cc_netplan_nm_patch.py index 49d6eb698d2..1c8b9e7dbd5 100644 --- a/cloudinit/config/cc_netplan_nm_patch.py +++ b/cloudinit/config/cc_netplan_nm_patch.py @@ -25,7 +25,7 @@ } -def exec_cmd(command: str) -> str | None: +def exec_cmd(command: list[str]) -> str | None: try: result = subp.subp(command) if result.stdout is not None: diff --git a/cloudinit/config/cc_rpi_connect.py b/cloudinit/config/cc_rpi_connect.py index 09e6aa0c301..fad63ffd842 100644 --- a/cloudinit/config/cc_rpi_connect.py +++ b/cloudinit/config/cc_rpi_connect.py @@ -24,7 +24,7 @@ def configure_rpi_connect(enable: bool) -> None: - LOG.debug(f"Configuring rpi-connect: {enable}") + LOG.debug("Configuring rpi-connect: %s", enable) num = 0 if enable else 1 diff --git a/cloudinit/config/cc_rpi_userdata.py b/cloudinit/config/cc_rpi_userdata.py index 2c19f711187..fd61a42446a 100644 --- a/cloudinit/config/cc_rpi_userdata.py +++ b/cloudinit/config/cc_rpi_userdata.py @@ -85,13 +85,13 @@ def run_userconf_service( if result.stderr: # Handle failure and restart if needed (Restart=on-failure logic) - LOG.debug(f"Userconf stderr service output: {result.stderr}") + LOG.debug("Userconf stderr service output: %s", result.stderr) return False else: lib_dir = os.path.dirname(MODULE_DEACTIVATION_FILE) # create deactivation file os.system( - f"mkdir -p {lib_dir} " "&& touch {MODULE_DEACTIVATION_FILE}" + f"mkdir -p {lib_dir} && touch {MODULE_DEACTIVATION_FILE}" ) LOG.debug("Userconf service completed successfully.") return True From 74c5111f79c6d489df98b61ff84066d33409f56b Mon Sep 17 00:00:00 2001 From: paulober Date: Wed, 11 Dec 2024 23:12:59 +0000 Subject: [PATCH 07/20] Simplify rpios integration Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- ...c_rpi_interfaces.py => cc_raspberry_pi.py} | 128 +++++----- cloudinit/config/cc_rpi_connect.py | 47 ---- cloudinit/config/cc_rpi_userdata.py | 228 ------------------ .../schemas/schema-cloud-config-v1.json | 151 +++++------- cloudinit/distros/raspberry-pi-os.py | 61 +++++ config/cloud.cfg.tmpl | 13 +- doc/module-docs/cc_raspberry_pi/data.yaml | 26 ++ doc/module-docs/cc_raspberry_pi/example1.yaml | 5 + .../example2.yaml | 0 .../example3.yaml | 0 .../example4.yaml | 0 doc/module-docs/cc_raspberry_pi/example5.yaml | 3 + doc/module-docs/cc_rpi_connect/data.yaml | 13 - doc/module-docs/cc_rpi_connect/example1.yaml | 2 - doc/module-docs/cc_rpi_interfaces/data.yaml | 19 -- .../cc_rpi_interfaces/example1.yaml | 4 - doc/module-docs/cc_rpi_userdata/data.yaml | 26 -- doc/module-docs/cc_rpi_userdata/example1.yaml | 2 - doc/module-docs/cc_rpi_userdata/example2.yaml | 3 - doc/module-docs/cc_rpi_userdata/example3.yaml | 4 - doc/rtd/reference/modules.rst | 8 +- pyproject.toml | 4 +- systemd/cloud-init-network.service.tmpl | 15 +- ..._interfaces.py => test_cc_raspberry_pi.py} | 18 +- tests/unittests/config/test_cc_rpi_connect.py | 76 ------ .../unittests/config/test_cc_rpi_userdata.py | 93 ------- tests/unittests/config/test_schema.py | 4 +- 27 files changed, 244 insertions(+), 709 deletions(-) rename cloudinit/config/{cc_rpi_interfaces.py => cc_raspberry_pi.py} (56%) delete mode 100644 cloudinit/config/cc_rpi_connect.py delete mode 100644 cloudinit/config/cc_rpi_userdata.py create mode 100644 doc/module-docs/cc_raspberry_pi/data.yaml create mode 100644 doc/module-docs/cc_raspberry_pi/example1.yaml rename doc/module-docs/{cc_rpi_interfaces => cc_raspberry_pi}/example2.yaml (100%) rename doc/module-docs/{cc_rpi_interfaces => cc_raspberry_pi}/example3.yaml (100%) rename doc/module-docs/{cc_rpi_interfaces => cc_raspberry_pi}/example4.yaml (100%) create mode 100644 doc/module-docs/cc_raspberry_pi/example5.yaml delete mode 100644 doc/module-docs/cc_rpi_connect/data.yaml delete mode 100644 doc/module-docs/cc_rpi_connect/example1.yaml delete mode 100644 doc/module-docs/cc_rpi_interfaces/data.yaml delete mode 100644 doc/module-docs/cc_rpi_interfaces/example1.yaml delete mode 100644 doc/module-docs/cc_rpi_userdata/data.yaml delete mode 100644 doc/module-docs/cc_rpi_userdata/example1.yaml delete mode 100644 doc/module-docs/cc_rpi_userdata/example2.yaml delete mode 100644 doc/module-docs/cc_rpi_userdata/example3.yaml rename tests/unittests/config/{test_cc_rpi_interfaces.py => test_cc_raspberry_pi.py} (85%) delete mode 100644 tests/unittests/config/test_cc_rpi_connect.py delete mode 100644 tests/unittests/config/test_cc_rpi_userdata.py diff --git a/cloudinit/config/cc_rpi_interfaces.py b/cloudinit/config/cc_raspberry_pi.py similarity index 56% rename from cloudinit/config/cc_rpi_interfaces.py rename to cloudinit/config/cc_raspberry_pi.py index d8fecf014b7..938bf529eda 100644 --- a/cloudinit/config/cc_rpi_interfaces.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2024, Raspberry Pi Ltd. +# Copyright (C) 2024-2025, Raspberry Pi Ltd. # # Author: Paul Oberosler # @@ -13,26 +13,38 @@ LOG = logging.getLogger(__name__) -RPI_INTERFACES_KEY = "rpi_interfaces" +RPI_BASE_KEY = "rpi" +RPI_INTERFACES_KEY = "interfaces" +ENABLE_RPI_CONNECT_KEY = "enable_rpi_connect" SUPPORTED_INTERFACES = { "spi": "do_spi", "i2c": "do_i2c", "serial": "do_serial", "onewire": "do_onewire", - "remote_gpio": "do_rgpio", - "ssh": "enable_ssh", + "remote_gpio": "do_rgpio" } RASPI_CONFIG_SERIAL_CONS_FN = "do_serial_cons" RASPI_CONFIG_SERIAL_HW_FN = "do_serial_hw" meta: MetaSchema = { - "id": "cc_rpi_interfaces", + "id": "cc_raspberry_pi", "distros": ["raspberry-pi-os"], "frequency": PER_INSTANCE, - "activate_by_schema_keys": [RPI_INTERFACES_KEY], + "activate_by_schema_keys": [RPI_BASE_KEY], } +def configure_rpi_connect(enable: bool) -> None: + LOG.debug("Configuring rpi-connect: %s", enable) + + num = 0 if enable else 1 + + try: + subp.subp(["/usr/bin/raspi-config", "do_rpi_connect", str(num)]) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure rpi-connect: %s", e) + + # TODO: test def require_reboot(cfg: Config) -> None: cfg["power_state"] = cfg.get("power_state", {}) @@ -93,22 +105,6 @@ def configure_serial_interface(cfg: dict | bool, instCfg: Config) -> None: LOG.error("Failed to configure serial console: %s", e) -def enable_ssh(cfg: Config, enable: bool) -> None: - if not enable: - return - - try: - subp.subp( - [ - "/usr/lib/raspberry-pi-sys-mods/imager_custom", - SUPPORTED_INTERFACES["ssh"], - ] - ) - require_reboot(cfg) - except subp.ProcessExecutionError as e: - LOG.error("Failed to enable ssh: %s", e) - - def configure_interface(iface: str, enable: bool) -> None: assert ( iface in SUPPORTED_INTERFACES.keys() and iface != "serial" @@ -128,53 +124,69 @@ def configure_interface(iface: str, enable: bool) -> None: def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if RPI_INTERFACES_KEY not in cfg: + if RPI_BASE_KEY not in cfg: return - elif not isinstance(cfg[RPI_INTERFACES_KEY], dict): + elif not isinstance(cfg[RPI_BASE_KEY], dict): LOG.warning( "Invalid value for %s: %s", - RPI_INTERFACES_KEY, - cfg[RPI_INTERFACES_KEY], + RPI_BASE_KEY, + cfg[RPI_BASE_KEY], ) return - elif not cfg[RPI_INTERFACES_KEY]: - LOG.debug("Empty value for %s. Skipping...", RPI_INTERFACES_KEY) + elif not cfg[RPI_BASE_KEY]: + LOG.debug("Empty value for %s. Skipping...", RPI_BASE_KEY) return - # check for supported ARM interfaces - for key in cfg[RPI_INTERFACES_KEY]: - if key not in SUPPORTED_INTERFACES.keys(): - LOG.warning("Invalid key for %s: %s", RPI_INTERFACES_KEY, key) - continue - - enable = cfg[RPI_INTERFACES_KEY][key] + for key in cfg[RPI_BASE_KEY]: + if key == ENABLE_RPI_CONNECT_KEY: + # expect it to be a dictionary + enable = cfg[ENABLE_RPI_CONNECT_KEY] - if key == "serial": - if not isinstance(enable, dict) and not isinstance(enable, bool): + if isinstance(enable, bool): + configure_rpi_connect(enable) + else: LOG.warning( - "Invalid value for %s.%s: %s", - RPI_INTERFACES_KEY, - key, - enable, + "Invalid value for %s: %s", ENABLE_RPI_CONNECT_KEY, enable ) - else: - configure_serial_interface(enable, cfg) continue - elif key == "ssh": - if not isinstance(enable, bool): + elif key == RPI_INTERFACES_KEY: + if not isinstance(cfg[RPI_BASE_KEY][key], dict): LOG.warning( - "Invalid value for %s.%s: %s", - RPI_INTERFACES_KEY, - key, - enable, + "Invalid value for %s: %s", + RPI_BASE_KEY, + cfg[RPI_BASE_KEY][key], ) - else: - enable_ssh(cfg, enable) - continue - - if isinstance(enable, bool): - configure_interface(key, enable) + return + elif not cfg[RPI_BASE_KEY][key]: + LOG.debug("Empty value for %s. Skipping...", key) + return + + # check for supported ARM interfaces + for subkey in cfg[RPI_BASE_KEY][key]: + if subkey not in SUPPORTED_INTERFACES.keys(): + LOG.warning("Invalid key for %s: %s", RPI_INTERFACES_KEY, subkey) + continue + + enable = cfg[RPI_INTERFACES_KEY][subkey] + + if subkey == "serial": + if not isinstance(enable, dict) and not isinstance(enable, bool): + LOG.warning( + "Invalid value for %s.%s: %s", + RPI_INTERFACES_KEY, + subkey, + enable, + ) + else: + configure_serial_interface(enable, cfg) + continue + + if isinstance(enable, bool): + configure_interface(subkey, enable) + else: + LOG.warning( + "Invalid value for %s.%s: %s", RPI_INTERFACES_KEY, subkey, enable + ) else: - LOG.warning( - "Invalid value for %s.%s: %s", RPI_INTERFACES_KEY, key, enable - ) + LOG.warning("Unsupported key: %s", key) + continue diff --git a/cloudinit/config/cc_rpi_connect.py b/cloudinit/config/cc_rpi_connect.py deleted file mode 100644 index fad63ffd842..00000000000 --- a/cloudinit/config/cc_rpi_connect.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2024, Raspberry Pi Ltd. -# -# Author: Paul Oberosler -# -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit import subp -from cloudinit.cloud import Cloud -from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema -from cloudinit.settings import PER_INSTANCE -import logging - - -LOG = logging.getLogger(__name__) -ENABLE_RPI_CONNECT_KEY = "enable_rpi_connect" - -meta: MetaSchema = { - "id": "cc_rpi_connect", - "distros": ["raspberry-pi-os"], - "frequency": PER_INSTANCE, - "activate_by_schema_keys": [ENABLE_RPI_CONNECT_KEY], -} - - -def configure_rpi_connect(enable: bool) -> None: - LOG.debug("Configuring rpi-connect: %s", enable) - - num = 0 if enable else 1 - - try: - subp.subp(["/usr/bin/raspi-config", "do_rpi_connect", str(num)]) - except subp.ProcessExecutionError as e: - LOG.error("Failed to configure rpi-connect: %s", e) - - -def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - if ENABLE_RPI_CONNECT_KEY in cfg: - # expect it to be a dictionary - enable = cfg[ENABLE_RPI_CONNECT_KEY] - - if isinstance(enable, bool): - configure_rpi_connect(enable) - else: - LOG.warning( - "Invalid value for %s: %s", ENABLE_RPI_CONNECT_KEY, enable - ) diff --git a/cloudinit/config/cc_rpi_userdata.py b/cloudinit/config/cc_rpi_userdata.py deleted file mode 100644 index fd61a42446a..00000000000 --- a/cloudinit/config/cc_rpi_userdata.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright (C) 2024, Raspberry Pi Ltd. -# -# Author: Paul Oberosler -# -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit import subp -from cloudinit.cloud import Cloud -from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema -from cloudinit.settings import PER_ALWAYS -import logging -import os -import subprocess -import time - -LOG = logging.getLogger(__name__) -DISABLE_PIWIZ_KEY = "disable_piwiz" -RPI_USERCONF_KEY = "rpi_userconf" -USERCONF_SERVICE_TTY = "/dev/tty8" -MODULE_DEACTIVATION_FILE = "/var/lib/userconf-pi/deactivate" - -meta: MetaSchema = { - "id": "cc_rpi_userdata", - "distros": ["raspberry-pi-os"], - # Run every boot to trigger setup wizard even when no settings - "frequency": PER_ALWAYS, - # "activate_by_schema_keys": [DISABLE_PIWIZ_KEY, RPI_USERCONF_KEY], - # When provided it would only start the module - # when the keys are present in the configuration - "activate_by_schema_keys": [], -} - - -def bool_to_str(value: bool | None) -> str: - return "Yes" if value else "No" - - -def get_fwloc_or_default() -> str: - fwloc = None - try: - # Run the command and capture the output - fwloc = subp.subp( - ["/usr/lib/raspberrypi-sys-mods/get_fw_loc"], decode="strict" - ).stdout.strip() - - # If the output is empty, set the default value - if not fwloc: - fwloc = "/boot" - except subp.ProcessExecutionError: - # If the command fails, set the default value - fwloc = "/boot" - return fwloc - - -def run_userconf_service( - base: str | None, passwd_override: str | None = None -) -> bool: - try: - # reset the TTY device - os.system(f"echo 'reset\\r\\n' > {USERCONF_SERVICE_TTY}") - - time.sleep(1) - # Execute the command on different tty - result = subp.subp( - [ - "openvt", - "-s", - "-f", - "-w", - "-c", - USERCONF_SERVICE_TTY[-1], - "--", - "/usr/lib/userconf-pi/userconf-service", - ], - timeout=(None if not passwd_override else 10), - decode="strict", - ) - - if base: - try: - os.remove(f"{base}/userconf.txt") - except FileNotFoundError: - pass - - if result.stderr: - # Handle failure and restart if needed (Restart=on-failure logic) - LOG.debug("Userconf stderr service output: %s", result.stderr) - return False - else: - lib_dir = os.path.dirname(MODULE_DEACTIVATION_FILE) - # create deactivation file - os.system( - f"mkdir -p {lib_dir} && touch {MODULE_DEACTIVATION_FILE}" - ) - LOG.debug("Userconf service completed successfully.") - return True - except subprocess.TimeoutExpired: - if base and os.path.exists(f"{base}/failed_userconf.txt"): - LOG.error("Invalid credentials provided for userconf-pi.") - os.remove(f"{base}/failed_userconf.txt") - else: - LOG.error("Userconf service timed out.") - return False - except Exception as e: - LOG.warning("Error running service: %s", e) - if base: - try: - os.remove(f"{base}/userconf.txt") - except FileNotFoundError: - pass - return False - - -def run_service( - passwd_override: str | None = None, user_override: str | None = None -) -> bool: - # Ensure the TTY exists before trying to open it - if not os.path.exists(USERCONF_SERVICE_TTY): - if not passwd_override: - LOG.error("TTY device %s does not exist.", USERCONF_SERVICE_TTY) - return False - else: - LOG.debug("TTY device %s does not exist.", USERCONF_SERVICE_TTY) - - # should never happen and not solvable by the user - assert (passwd_override is None and user_override is None) or ( - passwd_override is not None and user_override is not None - ), ( - "Internal error: User override is required when password " - "override is provided." - ) - - base: str | None = None - if passwd_override: - # write /boot/firmware/userconf.txt - # this will make userconf-service - # run silently with the provided credentials - base = get_fwloc_or_default() - assert base, "Internal error: Failed to get firmware location." - with open(f"{base}/userconf.txt", "w") as f: - f.write(f"{user_override}:{passwd_override}") - LOG.debug("Userconf override file written to %s/userconf.txt", base) - - LOG.debug("Start running userconf-pi service loop...") - while True: - if run_userconf_service(base, passwd_override): - break - # Wait for a moment before retrying - time.sleep(1) - LOG.debug("Userconf-pi service loop: retrying") - LOG.debug("Userconf-pi service loop finished.") - return True - - -def configure_pizwiz( - cfg: Config, - disable: bool, - passwd_override: str | None, - user_override: str | None = None, -) -> None: - LOG.debug( - "Configuring piwiz with disable_piwiz=%s, passwd_override=%s, " - "user_override=%s", - bool_to_str(disable), - bool_to_str(passwd_override is not None), - bool_to_str(user_override is not None), - ) - - if disable: - # execute cancel rename script to ensure - # piwiz isn't started (on desktop) - os.system("/usr/bin/cancel-rename pi") - else: - # execute userconf-pi service - # on desktop this doesn't have any effect - # as piwiz is started by the desktop environment - run_service(passwd_override, user_override) - - # populate users for other cloud-init modules to use - cfg["users"] = cfg.get("users", []) - if user_override and user_override not in cfg["users"]: - cfg["users"].append(user_override) - elif "pi" not in cfg["users"]: - cfg["users"].append("pi") - - -def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - disable_piwiz: bool = False - password_override: str | None = None - user_override: str | None = None - - if os.path.exists(MODULE_DEACTIVATION_FILE) or not os.path.exists( - "/usr/lib/userconf-pi" - ): - LOG.debug( - "Userconf-Pi: deactivation file detected or userconf-pi " - "not installed. Skipping..." - ) - return - - if RPI_USERCONF_KEY in cfg: - # expect it to be a dictionary - userconf = cfg[RPI_USERCONF_KEY] - - # look over miss configuration to - if isinstance(userconf, dict) and "password" in userconf: - password_override = userconf["password"] - # user key is optional with default to pi - user_override = userconf.get("user", "pi") - LOG.debug( - "Userconf override: user=%s, password=", - user_override, - ) - else: - LOG.error("Invalid userconf-pi configuration: %s", userconf) - - if not password_override and DISABLE_PIWIZ_KEY in cfg: - if isinstance(cfg[DISABLE_PIWIZ_KEY], bool): - disable_piwiz = cfg[DISABLE_PIWIZ_KEY] - else: - LOG.error( - "Invalid %s configuration: %s", - str(cfg[DISABLE_PIWIZ_KEY]), - DISABLE_PIWIZ_KEY, - ) - - configure_pizwiz(cfg, disable_piwiz, password_override, user_override) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index 0f345e74e7a..952fadca32d 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -35,7 +35,6 @@ "lxd", "mcollective", "mounts", - "netplan-nm-patch", "netplan_nm_patch", "ntp", "package-update-upgrade-install", @@ -45,6 +44,7 @@ "power-state-change", "power_state_change", "puppet", + "raspberry_pi", "reset-rmc", "reset_rmc", "resizefs", @@ -52,12 +52,6 @@ "resolv_conf", "rh-subscription", "rh_subscription", - "rpi-connect", - "rpi_connect", - "rpi-interfaces", - "rpi_interfaces", - "rpi-userdata", - "rpi_userdata", "rsyslog", "runcmd", "salt-minion", @@ -2656,93 +2650,70 @@ } } }, - "cc_rpi_connect": { + "cc_raspberry_pi": { "type": "object", "properties": { - "enable_rpi_connect": { - "type": "boolean", - "default": false, - "description": "Install and enable Raspberry Pi Connect. Default: ``false``." - } - } - }, - "cc_rpi_interfaces": { - "type": "object", - "properties": { - "rpi_interfaces": { + "rpi": { "type": "object", "properties": { - "spi": { - "type": "boolean", - "description": "Enable SPI interface. Default: ``false``.", - "default": false - }, - "i2c": { - "type": "boolean", - "description": "Enable I2C interface. Default: ``false``.", - "default": false - }, - "serial": { - "default": false, - "description": "Enable serial console. Default: ``false``.", - "oneOf": [ - { - "type": "boolean" + "rpi_interfaces": { + "type": "object", + "properties": { + "spi": { + "type": "boolean", + "description": "Enable SPI interface. Default: ``false``.", + "default": false }, - { - "type": "object", - "properties": { - "console": { - "type": "boolean", - "description": "Enable serial console. Default: ``false``.", - "default": false + "i2c": { + "type": "boolean", + "description": "Enable I2C interface. Default: ``false``.", + "default": false + }, + "serial": { + "default": false, + "description": "Enable serial console. Default: ``false``.", + "oneOf": [ + { + "type": "boolean" }, - "hardware": { - "type": "boolean", - "description": "Enable UART hardware. Default: ``false``.", - "default": false + { + "type": "object", + "properties": { + "console": { + "type": "boolean", + "description": "Enable serial console. Default: ``false``.", + "default": false + }, + "hardware": { + "type": "boolean", + "description": "Enable UART hardware. Default: ``false``.", + "default": false + } + } } - } + ] + }, + "onewire": { + "type": "boolean", + "description": "Enable 1-Wire interface. Default: ``false``.", + "default": false + }, + "remote_gpio": { + "type": "boolean", + "description": "Enable remote GPIO interface. Default: ``false``.", + "default": false + }, + "ssh": { + "type": "boolean", + "description": "Enable SSH. Default: ``false``.", + "default": false } - ] - }, - "onewire": { - "type": "boolean", - "description": "Enable 1-Wire interface. Default: ``false``.", - "default": false - }, - "remote_gpio": { - "type": "boolean", - "description": "Enable remote GPIO interface. Default: ``false``.", - "default": false + } }, - "ssh": { + "enable_rpi_connect": { "type": "boolean", - "description": "Enable SSH. Default: ``false``.", - "default": false - } - } - } - } - }, - "cc_rpi_userdata": { - "type": "object", - "properties": { - "disable_piwiz": { - "type": "boolean", - "default": false, - "description": "Manually disable the Raspberry Pi first boot wizard. Default: ``false``." - }, - "rpi_userconf": { - "type": "object", - "properties": { - "password": { - "type": "string", - "description": "The password for the default user." - }, - "user": { - "type": "string", - "description": "The username for the default user. Default: ``pi``." + "default": false, + "description": "Install and enable Raspberry Pi Connect. Default: ``false``." } } } @@ -4016,13 +3987,7 @@ "$ref": "#/$defs/cc_rh_subscription" }, { - "$ref": "#/$defs/cc_rpi_connect" - }, - { - "$ref": "#/$defs/cc_rpi_interfaces" - }, - { - "$ref": "#/$defs/cc_rpi_userdata" + "$ref": "#/$defs/cc_raspberry_pi" }, { "$ref": "#/$defs/cc_rsyslog" @@ -4125,7 +4090,6 @@ "disable_root_opts": {}, "disk_setup": {}, "drivers": {}, - "enable_rpi_connect": {}, "fan": {}, "final_message": {}, "fqdn": {}, @@ -4167,8 +4131,7 @@ "resize_rootfs": {}, "resolv_conf": {}, "rh_subscription": {}, - "rpi_interfaces": {}, - "rpi_userconf": {}, + "rpi": {}, "rsyslog": {}, "runcmd": {}, "salt_minion": {}, diff --git a/cloudinit/distros/raspberry-pi-os.py b/cloudinit/distros/raspberry-pi-os.py index c0bfa097e1a..c2a2b720a40 100644 --- a/cloudinit/distros/raspberry-pi-os.py +++ b/cloudinit/distros/raspberry-pi-os.py @@ -6,6 +6,8 @@ from cloudinit import subp from cloudinit.distros import debian +import os +import shutil class Distro(debian.Distro): @@ -43,3 +45,62 @@ def apply_locale(self, locale, out_fn=None, keyname="LANG"): f"{locale}.UTF-8", ] ) + + def add_user(self, name, **kwargs) -> bool: + """ + Add a user to the system using standard GNU tools + + This should be overridden on distros where useradd is not desirable or + not available. + + Returns False if user already exists, otherwise True. + """ + result = super().add_user(name, **kwargs) + + if not result: + return result + + try: + subp.subp( + [ + "/usr/bin/rename-user", + "-f", + "-s", + ], + update_env={"SUDO_USER": name}, + ) + + except subp.ProcessExecutionError as e: + self.log.error(f"Failed to setup user: {e}") + return False + + # Alacarte fixes + try: + # Ensure the sudoers directory exists + os.makedirs( + f"/home/{name}/.local/share/applications", + exist_ok=True) + os.makedirs( + f"/home/{name}/.local/share/desktop-directories", + exist_ok=True) + + stat_info = os.stat(f"/home/{name}") + uid = stat_info.st_uid + gid = stat_info.st_gid + + paths = [ + f"/home/{name}/.local", + f"/home/{name}/.local/share", + f"/home/{name}/.local/share/applications", + f"/home/{name}/.local/share/desktop-directories" + ] + + for path in paths: + shutil.chown(path, user=uid, group=gid) + os.chmod(path, 0o755) + + except Exception as e: + self.log.error(f"Failed to setup userhome: {e}") + return False + + return True diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 52e3a979bc6..dea7f0312a1 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -18,7 +18,7 @@ "gentoo": "users, wheel", "mariner": "wheel", "photon": "wheel", "openmandriva": "wheel, users, systemd-journal", - "raspberry-pi-os": "adm, dialout, cdrom, audio, users, sudo, video, games, plugdev, input, gpio, spi, i2c, netdev, render", + "raspberry-pi-os": "adm, dialout, cdrom, audio, users, sudo, video, games, plugdev, input, gpio, spi, i2c, netdev, render, lpadmin", "suse": "cdrom, users", "ubuntu": "adm, cdrom, dip, lxd, sudo", "unknown": "adm, cdrom, dip, lxd, sudo"}) %} @@ -137,14 +137,11 @@ cloud_init_modules: - ca_certs {% endif %} - rsyslog -{% if variant == "raspberry-pi-os" %} - - rpi_userdata -{% endif %} - users_groups - ssh - set_passwords {% if variant == "raspberry-pi-os" %} - - rpi_interfaces + - raspberry_pi {% endif %} # The modules that run in the 'config' stage @@ -165,9 +162,6 @@ cloud_config_modules: {% endif %} - locale {% endif %} -{% if variant == "raspberry-pi-os" %} - - rpi_interfaces -{% endif %} {% if variant == "alpine" %} - apk_configure {% elif variant in ["debian", "ubuntu", "unknown"] %} @@ -216,9 +210,6 @@ cloud_final_modules: - chef {% endif %} - ansible -{% if variant == "raspberry-pi-os" %} - - rpi_connect -{% endif %} {% if variant not in ["azurelinux"] %} - mcollective - salt_minion diff --git a/doc/module-docs/cc_raspberry_pi/data.yaml b/doc/module-docs/cc_raspberry_pi/data.yaml new file mode 100644 index 00000000000..b7882bc1a95 --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/data.yaml @@ -0,0 +1,26 @@ +cc_raspberry_pi: + description: | + This module handles ARM interface configuration for Raspberry Pi. + + It also handles Raspberry Pi Connect installation and enablement. + Raspberry Pi Connect service will be installed and enabled to auto start on boot. + + This only works on Raspberry Pi OS (bookworm and later). + examples: + - comment: > + This example will enable the SPI and I2C interfaces on Raspberry Pi. + file: cc_raspberry_pi/example1.yaml + - comment: > + This example will enable the serial interface on Raspberry Pi. + file: cc_raspberry_pi/example2.yaml + - comment: > + This example will enable the serial interface on Raspberry Pi 5 and disable the UART hardware while enabling the console. + file: cc_raspberry_pi/example3.yaml + - comment: > + This example will enable ssh and the UART hardware without binding it to the console. + file: cc_raspberry_pi/example4.yaml + - comment: > + This example will enable the Raspberry Pi Connect service. + file: cc_raspberry_pi/example5.yaml + name: Raspberry Pi Configuration + title: Configure Raspberry Pi ARM interfaces and enable Raspberry Pi Connect diff --git a/doc/module-docs/cc_raspberry_pi/example1.yaml b/doc/module-docs/cc_raspberry_pi/example1.yaml new file mode 100644 index 00000000000..0fe2ac83f91 --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example1.yaml @@ -0,0 +1,5 @@ +#cloud-config +rpi: + interfaces: + spi: true + i2c: true diff --git a/doc/module-docs/cc_rpi_interfaces/example2.yaml b/doc/module-docs/cc_raspberry_pi/example2.yaml similarity index 100% rename from doc/module-docs/cc_rpi_interfaces/example2.yaml rename to doc/module-docs/cc_raspberry_pi/example2.yaml diff --git a/doc/module-docs/cc_rpi_interfaces/example3.yaml b/doc/module-docs/cc_raspberry_pi/example3.yaml similarity index 100% rename from doc/module-docs/cc_rpi_interfaces/example3.yaml rename to doc/module-docs/cc_raspberry_pi/example3.yaml diff --git a/doc/module-docs/cc_rpi_interfaces/example4.yaml b/doc/module-docs/cc_raspberry_pi/example4.yaml similarity index 100% rename from doc/module-docs/cc_rpi_interfaces/example4.yaml rename to doc/module-docs/cc_raspberry_pi/example4.yaml diff --git a/doc/module-docs/cc_raspberry_pi/example5.yaml b/doc/module-docs/cc_raspberry_pi/example5.yaml new file mode 100644 index 00000000000..3d5354ab3db --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example5.yaml @@ -0,0 +1,3 @@ +#cloud-config +rpi: + enable_rpi_connect: true diff --git a/doc/module-docs/cc_rpi_connect/data.yaml b/doc/module-docs/cc_rpi_connect/data.yaml deleted file mode 100644 index c738697c4f3..00000000000 --- a/doc/module-docs/cc_rpi_connect/data.yaml +++ /dev/null @@ -1,13 +0,0 @@ -cc_rpi_connect: - description: | - This module handles Raspberry Pi Connect installation and enablement. - It will install the Raspberry Pi Connect service and enable it to run on boot. - - Supported operating systems: - - Raspberry Pi OS (bookworm and later) - examples: - - comment: > - This example will enable the Raspberry Pi Connect service. - file: cc_rpi_connect/example1.yaml - name: Raspberry Pi Connect - title: Install and enable Raspberry Pi Connect service diff --git a/doc/module-docs/cc_rpi_connect/example1.yaml b/doc/module-docs/cc_rpi_connect/example1.yaml deleted file mode 100644 index 61b46e62b13..00000000000 --- a/doc/module-docs/cc_rpi_connect/example1.yaml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -enable_rpi_connect: true diff --git a/doc/module-docs/cc_rpi_interfaces/data.yaml b/doc/module-docs/cc_rpi_interfaces/data.yaml deleted file mode 100644 index 168ee408c81..00000000000 --- a/doc/module-docs/cc_rpi_interfaces/data.yaml +++ /dev/null @@ -1,19 +0,0 @@ -cc_rpi_interfaces: - description: | - This module handles ARM interface configuration for Raspberry Pi. - I also has an option to enable the ssh service which is disabled by default. - examples: - - comment: > - This example will enable the SPI and I2C interfaces on Raspberry Pi. - file: cc_rpi_interfaces/example1.yaml - - comment: > - This example will enable the serial interface on Raspberry Pi. - file: cc_rpi_interfaces/example2.yaml - - comment: > - This example will enable the serial interface on Raspberry Pi 5 and disable the UART hardware while enabling the console. - file: cc_rpi_interfaces/example3.yaml - - comment: > - This example will enable ssh and the UART hardware without binding it to the console. - file: cc_rpi_interfaces/example4.yaml - name: Raspberry Pi Interfaces - title: Configure Raspberry Pi ARM interfaces diff --git a/doc/module-docs/cc_rpi_interfaces/example1.yaml b/doc/module-docs/cc_rpi_interfaces/example1.yaml deleted file mode 100644 index 68a0268851a..00000000000 --- a/doc/module-docs/cc_rpi_interfaces/example1.yaml +++ /dev/null @@ -1,4 +0,0 @@ -#cloud-config -rpi_interfaces: - spi: true - i2c: true diff --git a/doc/module-docs/cc_rpi_userdata/data.yaml b/doc/module-docs/cc_rpi_userdata/data.yaml deleted file mode 100644 index 04a9ebc3ee1..00000000000 --- a/doc/module-docs/cc_rpi_userdata/data.yaml +++ /dev/null @@ -1,26 +0,0 @@ -cc_rpi_userdata: - description: | - A cloud-init module that is intented to run in the final stage of the cloud-init process. - - It will take care of configuring and triggering the setup wizards on Raspberry Pi OS. - - On desktop images, it will configure the setup wizard (piwiz) or - disable it if requested. - - On lite images, it will configure/disable the userconf-pi package - and trigger the userconf-pi setup wizard. - - Supported operating systems: - - Raspberry Pi OS (bookworm and later) - examples: - - comment: > - This example will manually disable the setup wizard. - file: cc_rpi_userdata/example1.yaml - - comment: > - This example will set a password for the default pi user. (disables the setup wizard) - file: cc_rpi_userdata/example2.yaml - - comment: > - This example will trigger a rename of the default pi user and set a password. (disables the setup wizard) - file: cc_rpi_userdata/example3.yaml - name: Raspberry Pi OS Userdata - title: Configure and trigger Raspberry Pi OS setup wizards diff --git a/doc/module-docs/cc_rpi_userdata/example1.yaml b/doc/module-docs/cc_rpi_userdata/example1.yaml deleted file mode 100644 index e6b284d7b73..00000000000 --- a/doc/module-docs/cc_rpi_userdata/example1.yaml +++ /dev/null @@ -1,2 +0,0 @@ -#cloud-config -disable_piwiz: true diff --git a/doc/module-docs/cc_rpi_userdata/example2.yaml b/doc/module-docs/cc_rpi_userdata/example2.yaml deleted file mode 100644 index 93d9edb8348..00000000000 --- a/doc/module-docs/cc_rpi_userdata/example2.yaml +++ /dev/null @@ -1,3 +0,0 @@ -#cloud-config -rpi_userconf: - password: "my_super_secret_password_hash" diff --git a/doc/module-docs/cc_rpi_userdata/example3.yaml b/doc/module-docs/cc_rpi_userdata/example3.yaml deleted file mode 100644 index 3b98695de3c..00000000000 --- a/doc/module-docs/cc_rpi_userdata/example3.yaml +++ /dev/null @@ -1,4 +0,0 @@ -#cloud-config -rpi_userconf: - user: "my_new_user_name" - password: "my_super_secret_password_hash" diff --git a/doc/rtd/reference/modules.rst b/doc/rtd/reference/modules.rst index aec67115598..ffa541d1508 100644 --- a/doc/rtd/reference/modules.rst +++ b/doc/rtd/reference/modules.rst @@ -76,18 +76,14 @@ Modules :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_puppet/data.yaml :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_raspberry_pi/data.yaml + :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_resizefs/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_resolv_conf/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_rh_subscription/data.yaml :template: modules.tmpl -.. datatemplate:yaml:: ../../module-docs/cc_rpi_connect/data.yaml - :template: modules.tmpl -.. datatemplate:yaml:: ../../module-docs/cc_rpi_interfaces/data.yaml - :template: modules.tmpl -.. datatemplate:yaml:: ../../module-docs/cc_rpi_userdata/data.yaml - :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_rsyslog/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_runcmd/data.yaml diff --git a/pyproject.toml b/pyproject.toml index 0c15283edbc..55d244691f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,12 +129,10 @@ module = [ "tests.unittests.config.test_cc_mcollective", "tests.unittests.config.test_cc_phone_home", "tests.unittests.config.test_cc_puppet", + "tests.unittests.config.test_cc_raspberry_pi", "tests.unittests.config.test_cc_resizefs", "tests.unittests.config.test_cc_resolv_conf", "tests.unittests.config.test_cc_rh_subscription", - "tests.unittests.config.test_cc_rpi_connect", - "tests.unittests.config.test_cc_rpi_interfaces", - "tests.unittests.config.test_cc_rpi_userdata", "tests.unittests.config.test_cc_rsyslog", "tests.unittests.config.test_cc_runcmd", "tests.unittests.config.test_cc_snap", diff --git a/systemd/cloud-init-network.service.tmpl b/systemd/cloud-init-network.service.tmpl index d005f8fed0f..61425b4a9fd 100644 --- a/systemd/cloud-init-network.service.tmpl +++ b/systemd/cloud-init-network.service.tmpl @@ -2,7 +2,7 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Network Stage -{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} +{% if variant not in ["almalinux", "cloudlinux", "photon", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=cloud-init-local.service @@ -14,14 +14,10 @@ After=systemd-networkd-wait-online.service {% endif %} {% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} After=networking.service -{% if variant == "raspberry-pi-os" %} -Wants=NetworkManager-wait-online.service -After=NetworkManager-wait-online.service -{% endif %} {% endif %} {% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", - "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", - "suse", "TencentOS", "virtuozzo"] %} + "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "raspberry-pi-os", + "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} After=NetworkManager.service After=NetworkManager-wait-online.service {% endif %} @@ -38,11 +34,8 @@ Before=avahi-daemon.service Before=sshd-keygen.service Before=sshd.service Before=systemd-user-sessions.service -{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} -{% if variant not in ["raspberry-pi-os"] %} -{# stops cloud-init-network from starting on rpios #} +{% if variant in ["ubuntu", "unknown", "debian"] %} Before=sysinit.target -{% endif %} Before=shutdown.target Conflicts=shutdown.target {% endif %} diff --git a/tests/unittests/config/test_cc_rpi_interfaces.py b/tests/unittests/config/test_cc_raspberry_pi.py similarity index 85% rename from tests/unittests/config/test_cc_rpi_interfaces.py rename to tests/unittests/config/test_cc_raspberry_pi.py index c9d55db2de6..65a23d21ea3 100644 --- a/tests/unittests/config/test_cc_rpi_interfaces.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config.cc_rpi_interfaces import RPI_INTERFACES_KEY +from cloudinit.config.cc_raspberry_pi import RPI_BASE_KEY, RPI_INTERFACES_KEY, ENABLE_RPI_CONNECT_KEY from cloudinit.config.schema import ( SchemaValidationError, get_schema, @@ -93,27 +93,33 @@ def test_get_enabled_interfaces(self, mock_subp, mock_path_exists): @skipUnlessJsonSchema() -class TestCCRPiInterfacesSchema: +class TestCCRPiSchema: @pytest.mark.parametrize( "config, error_msg", [ - ({RPI_INTERFACES_KEY: {"spi": True, "i2c": False}}, None), + ({RPI_BASE_KEY: { RPI_INTERFACES_KEY: {"spi": True, "i2c": False}} }, None), ( - {RPI_INTERFACES_KEY: {"spi": "true"}}, + {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"spi": "true"}}}, "'true' is not of type 'boolean'", ), ( + {RPI_BASE_KEY: { RPI_INTERFACES_KEY: { "serial": {"console": True, "hardware": False} } - }, + }}, None, ), ( - {RPI_INTERFACES_KEY: {"serial": {"console": 123}}}, + {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": {"console": 123}}} }, "123 is not of type 'boolean'", ), + ({RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}}, None), + ( + {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: "true"}}, + "'true' is not of type 'boolean'", + ), ], ) def test_schema_validation(self, config, error_msg): diff --git a/tests/unittests/config/test_cc_rpi_connect.py b/tests/unittests/config/test_cc_rpi_connect.py deleted file mode 100644 index 19284e49864..00000000000 --- a/tests/unittests/config/test_cc_rpi_connect.py +++ /dev/null @@ -1,76 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit.config.cc_rpi_connect import ENABLE_RPI_CONNECT_KEY -from cloudinit.config.schema import ( - SchemaValidationError, - get_schema, - validate_cloudconfig_schema, -) -from tests.unittests.helpers import skipUnlessJsonSchema -import pytest - - -""" -def is_notPi() -> bool: - \"""Most tests are Raspberry Pi OS only.\""" - return not os.path.exists("/etc/rpi-issue") - -@mock.patch('cloudinit.subp.subp') -class TestCCRPiConnect(CiTestCase): - \"""Tests work in progress. Just partially implemented to - show the idea.\""" - - @mock.patch('cloudinit.subp.subp') - def test_configure_rpi_connect_enabled(self, mock_subp): - if is_notPi(): - return - config = {ENABLE_RPI_CONNECT_KEY: True} - handle("cc_rpi_connect", config, mock.Mock(), []) - mock_subp.assert_called_with( - ['/usr/bin/raspi-config', 'do_rpi_connect', '0']) - - @mock.patch('cloudinit.subp.subp') - def test_configure_rpi_connect_disabled(self, mock_subp): - if is_notPi(): - return - config = {ENABLE_RPI_CONNECT_KEY: False} - handle("cc_rpi_connect", config, mock.Mock(), []) - mock_subp.assert_called_with( - ['/usr/bin/raspi-config', 'do_rpi_connect', '1']) - - @mock.patch('os.path.exists') - def test_rpi_connect_installed(self, mock_path_exists): - if is_notPi(): - return - # Simulate rpi-connect is installed - mock_path_exists.return_value = True - assert mock_path_exists('/usr/bin/rpi-connect') - - @mock.patch('os.path.exists') - def test_rpi_connect_not_installed(self, mock_path_exists): - if is_notPi(): - return - # Simulate rpi-connect is not installed - mock_path_exists.return_value = False - assert not mock_path_exists('/usr/bin/rpi-connect') -""" - - -@skipUnlessJsonSchema() -class TestCCRPiConnectSchema: - @pytest.mark.parametrize( - "config, error_msg", - [ - ({ENABLE_RPI_CONNECT_KEY: True}, None), - ( - {ENABLE_RPI_CONNECT_KEY: "true"}, - "'true' is not of type 'boolean'", - ), - ], - ) - def test_schema_validation(self, config, error_msg): - if error_msg is None: - validate_cloudconfig_schema(config, get_schema(), strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_rpi_userdata.py b/tests/unittests/config/test_cc_rpi_userdata.py deleted file mode 100644 index 75d78b05018..00000000000 --- a/tests/unittests/config/test_cc_rpi_userdata.py +++ /dev/null @@ -1,93 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit.config.cc_rpi_userdata import ( - DISABLE_PIWIZ_KEY, - RPI_USERCONF_KEY, -) -from cloudinit.config.schema import ( - SchemaValidationError, - get_schema, - validate_cloudconfig_schema, -) -import pytest -from tests.unittests.helpers import skipUnlessJsonSchema - -""" -def is_notPi() -> bool: - \"""Most tests are Raspberry Pi OS only.\""" - return not os.path.exists("/etc/rpi-issue") - -@mock.patch('cloudinit.subp.subp') -class TestCCRPiUserdata(CiTestCase): - \"""Tests work in progress. Just partially implemented - to show the idea.\""" - - @mock.patch('subprocess.run') - def test_userconf_service_runs(self, mock_run): - if is_notPi(): - return - mock_run.return_value.returncode = 0 - result = run_service('hashedpassword', 'newuser') - assert result is True - - @mock.patch('subprocess.run') - def test_userconf_service_fails(self, mock_run): - if is_notPi(): - return - mock_run.return_value.returncode = 1 - result = run_service('hashedpassword', 'newuser') - assert result is False - - @mock.patch('os.path.exists') - def test_check_piwiz_disabled(self, mock_path_exists): - if is_notPi(): - return - mock_path_exists.side_effect = [False, False, False] - assert not mock_path_exists('/var/lib/userconf-pi/autologin') - assert not mock_path_exists('/etc/ssh/sshd_config.d/rename_user.conf') - assert not mock_path_exists('/etc/xdg/autostart/piwiz.desktop') - - @mock.patch('os.listdir') - def test_check_default_user_renamed(self, mock_listdir): - if is_notPi(): - return - mock_listdir.return_value = ['newuser'] - assert 'newuser' in os.listdir('/home') - - @mock.patch('os.listdir') - def test_default_user_still_exists(self, mock_listdir): - if is_notPi(): - return - mock_listdir.return_value = ['pi'] - assert 'pi' in os.listdir('/home') -""" - - -@skipUnlessJsonSchema() -class TestCCRPiUserdataSchema: - @pytest.mark.parametrize( - "config, error_msg", - [ - ({DISABLE_PIWIZ_KEY: True}, None), - ( - { - RPI_USERCONF_KEY: { - "password": "hashedpassword", - "user": "newuser", - } - }, - None, - ), - ({DISABLE_PIWIZ_KEY: "true"}, "'true' is not of type 'boolean'"), - ( - {RPI_USERCONF_KEY: {"password": 12345}}, - "12345 is not of type 'string'", - ), - ], - ) - def test_schema_validation(self, config, error_msg): - if error_msg is None: - validate_cloudconfig_schema(config, get_schema(), strict=True) - else: - with pytest.raises(SchemaValidationError, match=error_msg): - validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 36c47e8f17e..eb7b6727a3b 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -255,12 +255,10 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_phone_home"}, {"$ref": "#/$defs/cc_power_state_change"}, {"$ref": "#/$defs/cc_puppet"}, + {"$ref": "#/$defs/cc_raspberry_pi"}, {"$ref": "#/$defs/cc_resizefs"}, {"$ref": "#/$defs/cc_resolv_conf"}, {"$ref": "#/$defs/cc_rh_subscription"}, - {"$ref": "#/$defs/cc_rpi_connect"}, - {"$ref": "#/$defs/cc_rpi_interfaces"}, - {"$ref": "#/$defs/cc_rpi_userdata"}, {"$ref": "#/$defs/cc_rsyslog"}, {"$ref": "#/$defs/cc_runcmd"}, {"$ref": "#/$defs/cc_salt_minion"}, From 9a0cb1c97fa104f02263d02070780d7a558fbf94 Mon Sep 17 00:00:00 2001 From: paulober Date: Mon, 7 Apr 2025 23:22:14 +0200 Subject: [PATCH 08/20] Remove netplan_nm_patch Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_netplan_nm_patch.py | 129 ------------------ .../schemas/schema-cloud-config-v1.json | 8 -- doc/module-docs/cc_netplan_nm_patch/data.yaml | 13 -- doc/rtd/reference/modules.rst | 2 - tests/unittests/config/test_schema.py | 1 - 5 files changed, 153 deletions(-) delete mode 100644 cloudinit/config/cc_netplan_nm_patch.py delete mode 100644 doc/module-docs/cc_netplan_nm_patch/data.yaml diff --git a/cloudinit/config/cc_netplan_nm_patch.py b/cloudinit/config/cc_netplan_nm_patch.py deleted file mode 100644 index 1c8b9e7dbd5..00000000000 --- a/cloudinit/config/cc_netplan_nm_patch.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (C) 2024, Raspberry Pi Ltd. -# -# Author: Paul Oberosler -# -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit import subp -from cloudinit.cloud import Cloud -from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema -from cloudinit.distros import ALL_DISTROS -from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE -from cloudinit.settings import PER_INSTANCE -import logging -import os -import re - -LOG = logging.getLogger(__name__) - -meta: MetaSchema = { - "id": "cc_netplan_nm_patch", - "distros": [ALL_DISTROS], - "frequency": PER_INSTANCE, - "activate_by_schema_keys": [], -} - - -def exec_cmd(command: list[str]) -> str | None: - try: - result = subp.subp(command) - if result.stdout is not None: - return result.stdout - except subp.ProcessExecutionError as e: - LOG.error("Failed to execute command: %s", e) - return None - LOG.debug("Command has no stdout: %s", command) - return None - - -def get_netplan_generated_configs() -> list[str]: - """Get the UUIDs of all connections starting with 'netplan-'.""" - output = exec_cmd(["nmcli", "connection", "show"]) - if output is None: - return [] - - netplan_conns = [] - for line in output.splitlines(): - if line.startswith("netplan-"): - parts = line.split() - if len(parts) > 1: - # name = parts[0] - uuid = parts[1] - netplan_conns.append(uuid) - return netplan_conns - - -def get_connection_object_path(uuid: str) -> str | None: - """Get the D-Bus object path for a connection by UUID.""" - output = exec_cmd( - [ - "busctl", - "call", - "org.freedesktop.NetworkManager", - "/org/freedesktop/NetworkManager/Settings", - "org.freedesktop.NetworkManager.Settings", - "GetConnectionByUuid", - "s", - uuid, - ] - ) - - path_match = ( - re.search( - r'o\s+"(/org/freedesktop/NetworkManager/Settings/\d+)"', output - ) - if output - else None - ) - if path_match: - return path_match.group(1) - else: - LOG.error("Failed to find object path for connection: %s", uuid) - return None - - -def save_connection(obj_path: str) -> None: - """Call the Save method on the D-Bus obj path for a connection.""" - result = exec_cmd( - [ - "busctl", - "call", - "org.freedesktop.NetworkManager", - obj_path, - "org.freedesktop.NetworkManager.Settings.Connection", - "Save", - ] - ) - - if result is None: - LOG.error("Failed to save connection: %s", obj_path) - else: - LOG.debug("Saved connection: %s", obj_path) - - -def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: - LOG.debug("Applying netplan patch") - - # remove cloud-init file after NetworkManager has generated - # replacement netplan configurations to avoid conflicts in the - # future - - try: - np_conns = get_netplan_generated_configs() - if not np_conns: - LOG.debug("No netplan connections found") - return - - for conn_uuid in np_conns: - obj_path = get_connection_object_path(conn_uuid) - if obj_path is None: - continue - save_connection(obj_path) - - os.remove(CLOUDINIT_NETPLAN_FILE) - LOG.debug("Netplan cfg has been patched: %s", CLOUDINIT_NETPLAN_FILE) - except subp.ProcessExecutionError as e: - LOG.error("Failed to patch netplan cfg: %s", e) - except Exception as e: - LOG.error("Failed to patch netplan cfg: %s", e) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index 952fadca32d..5f3168d422d 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -35,7 +35,6 @@ "lxd", "mcollective", "mounts", - "netplan_nm_patch", "ntp", "package-update-upgrade-install", "package_update_upgrade_install", @@ -2137,10 +2136,6 @@ } } }, - "cc_netplan_nm_patch": { - "type": "object", - "properties": {} - }, "cc_ntp": { "type": "object", "properties": { @@ -3959,9 +3954,6 @@ { "$ref": "#/$defs/cc_mounts" }, - { - "$ref": "#/$defs/cc_netplan_nm_patch" - }, { "$ref": "#/$defs/cc_ntp" }, diff --git a/doc/module-docs/cc_netplan_nm_patch/data.yaml b/doc/module-docs/cc_netplan_nm_patch/data.yaml deleted file mode 100644 index f07c41688d6..00000000000 --- a/doc/module-docs/cc_netplan_nm_patch/data.yaml +++ /dev/null @@ -1,13 +0,0 @@ -cc_netplan_nm_patch: - description: | - This module resolves the issue where if you edit a netplan generated network configuration - in NetworkManager, the changes are saved to netplan as new configuration files. - This will cause netplan overwrite the NetworkManager generated netplan configuration on the next boot. - - This module will give NetworkManager a hint to generated netplan configuration files - and then removes the cloud-init generated netplan configuration file. - - This module expects that NetworkManager has the netplan integration patch applied. - examples: - name: Netplan NetworkManager Patch - title: Patch NetworkManager netplan interop issue diff --git a/doc/rtd/reference/modules.rst b/doc/rtd/reference/modules.rst index ffa541d1508..5ce7134de79 100644 --- a/doc/rtd/reference/modules.rst +++ b/doc/rtd/reference/modules.rst @@ -64,8 +64,6 @@ Modules :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_mounts/data.yaml :template: modules.tmpl -.. datatemplate:yaml:: ../../module-docs/cc_netplan_nm_patch/data.yaml - :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_ntp/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_package_update_upgrade_install/data.yaml diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index eb7b6727a3b..fb16a491939 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -249,7 +249,6 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_lxd"}, {"$ref": "#/$defs/cc_mcollective"}, {"$ref": "#/$defs/cc_mounts"}, - {"$ref": "#/$defs/cc_netplan_nm_patch"}, {"$ref": "#/$defs/cc_ntp"}, {"$ref": "#/$defs/cc_package_update_upgrade_install"}, {"$ref": "#/$defs/cc_phone_home"}, From bb0f7c83aecad48dbff6b0e6c60b6d885bd21a95 Mon Sep 17 00:00:00 2001 From: paulober Date: Tue, 8 Apr 2025 05:39:32 +0200 Subject: [PATCH 09/20] Remove left-over Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/schemas/schema-cloud-config-v1.json | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index 5f3168d422d..29497533e1a 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -4077,7 +4077,6 @@ "create_hostname_file": {}, "device_aliases": {}, "disable_ec2_metadata": {}, - "disable_piwiz": {}, "disable_root": {}, "disable_root_opts": {}, "disk_setup": {}, From aa3a9c77bbf4a5e0ad16636defa5fce8c2a1df67 Mon Sep 17 00:00:00 2001 From: paulober Date: Tue, 8 Apr 2025 16:58:51 +0200 Subject: [PATCH 10/20] Fix tox errors in PR files Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 19 ++- cloudinit/distros/raspberry-pi-os.py | 33 ++--- .../unittests/config/test_cc_raspberry_pi.py | 115 ++++-------------- tests/unittests/test_render_template.py | 5 +- 4 files changed, 56 insertions(+), 116 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index 938bf529eda..eee98ad87fa 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -4,13 +4,13 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import logging + from cloudinit import subp from cloudinit.cloud import Cloud from cloudinit.config import Config from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE -import logging - LOG = logging.getLogger(__name__) RPI_BASE_KEY = "rpi" @@ -21,7 +21,7 @@ "i2c": "do_i2c", "serial": "do_serial", "onewire": "do_onewire", - "remote_gpio": "do_rgpio" + "remote_gpio": "do_rgpio", } RASPI_CONFIG_SERIAL_CONS_FN = "do_serial_cons" RASPI_CONFIG_SERIAL_HW_FN = "do_serial_hw" @@ -164,13 +164,17 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: # check for supported ARM interfaces for subkey in cfg[RPI_BASE_KEY][key]: if subkey not in SUPPORTED_INTERFACES.keys(): - LOG.warning("Invalid key for %s: %s", RPI_INTERFACES_KEY, subkey) + LOG.warning( + "Invalid key for %s: %s", RPI_INTERFACES_KEY, subkey + ) continue enable = cfg[RPI_INTERFACES_KEY][subkey] if subkey == "serial": - if not isinstance(enable, dict) and not isinstance(enable, bool): + if not isinstance(enable, dict) and not isinstance( + enable, bool + ): LOG.warning( "Invalid value for %s.%s: %s", RPI_INTERFACES_KEY, @@ -185,7 +189,10 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: configure_interface(subkey, enable) else: LOG.warning( - "Invalid value for %s.%s: %s", RPI_INTERFACES_KEY, subkey, enable + "Invalid value for %s.%s: %s", + RPI_INTERFACES_KEY, + subkey, + enable, ) else: LOG.warning("Unsupported key: %s", key) diff --git a/cloudinit/distros/raspberry-pi-os.py b/cloudinit/distros/raspberry-pi-os.py index c2a2b720a40..60164bcf14b 100644 --- a/cloudinit/distros/raspberry-pi-os.py +++ b/cloudinit/distros/raspberry-pi-os.py @@ -4,11 +4,12 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import subp -from cloudinit.distros import debian import os import shutil +from cloudinit import subp +from cloudinit.distros import debian + class Distro(debian.Distro): def __init__(self, name, cfg, paths): @@ -59,7 +60,7 @@ def add_user(self, name, **kwargs) -> bool: if not result: return result - + try: subp.subp( [ @@ -69,30 +70,30 @@ def add_user(self, name, **kwargs) -> bool: ], update_env={"SUDO_USER": name}, ) - + except subp.ProcessExecutionError as e: - self.log.error(f"Failed to setup user: {e}") + self.log.error("Failed to setup user:", e) return False # Alacarte fixes try: # Ensure the sudoers directory exists os.makedirs( - f"/home/{name}/.local/share/applications", - exist_ok=True) + f"/home/{name}/.local/share/applications", exist_ok=True + ) os.makedirs( - f"/home/{name}/.local/share/desktop-directories", - exist_ok=True) - + f"/home/{name}/.local/share/desktop-directories", exist_ok=True + ) + stat_info = os.stat(f"/home/{name}") uid = stat_info.st_uid gid = stat_info.st_gid - paths = [ - f"/home/{name}/.local", - f"/home/{name}/.local/share", - f"/home/{name}/.local/share/applications", - f"/home/{name}/.local/share/desktop-directories" + paths = [ + f"/home/{name}/.local", + f"/home/{name}/.local/share", + f"/home/{name}/.local/share/applications", + f"/home/{name}/.local/share/desktop-directories", ] for path in paths: @@ -100,7 +101,7 @@ def add_user(self, name, **kwargs) -> bool: os.chmod(path, 0o755) except Exception as e: - self.log.error(f"Failed to setup userhome: {e}") + self.log.error("Failed to setup userhome:", e) return False return True diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index 65a23d21ea3..4d6d1346089 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -1,95 +1,18 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config.cc_raspberry_pi import RPI_BASE_KEY, RPI_INTERFACES_KEY, ENABLE_RPI_CONNECT_KEY +import pytest + +from cloudinit.config.cc_raspberry_pi import ( + ENABLE_RPI_CONNECT_KEY, + RPI_BASE_KEY, + RPI_INTERFACES_KEY, +) from cloudinit.config.schema import ( SchemaValidationError, get_schema, validate_cloudconfig_schema, ) from tests.unittests.helpers import skipUnlessJsonSchema -import pytest - - -""" -def is_notPi() -> bool: - \"""Most tests are Raspberry Pi OS only.\""" - return not os.path.exists("/etc/rpi-issue") - -@mock.patch('cloudinit.subp.subp') -class TestCCRPiInterfaces(CiTestCase): - \"""Tests work in progress. Just partially implemented to - show the idea.\""" - - @mock.patch('cloudinit.subp.subp') - def test_configure_spi_interface(self, mock_subp): - if is_notPi(): - return - config = { - RPI_INTERFACES_KEY: { - "spi": True - } - } - handle("cc_rpi_interfaces", config, mock.Mock(), []) - mock_subp.assert_called_with([ - '/usr/bin/raspi-config', - 'nonint', - SUPPORTED_INTERFACES["spi"], - '0']) - - @mock.patch('cloudinit.subp.subp') - def test_configure_serial_interface_as_dict(self, mock_subp): - if is_notPi(): - return - config = { - RPI_INTERFACES_KEY: { - "serial": { - "console": True, - "hardware": False - } - } - } - handle("cc_rpi_interfaces", config, mock.Mock(), []) - mock_subp.assert_any_call([ - '/usr/bin/raspi-config', 'nonint', 'do_serial_cons', '0']) - - @mock.patch('cloudinit.subp.subp') - def test_configure_invalid_interface(self, mock_subp): - if is_notPi(): - return - config = { - RPI_INTERFACES_KEY: { - "unknown_interface": True - } - } - handle("cc_rpi_interfaces", config, mock.Mock(), []) - mock_subp.assert_not_called() - - @mock.patch('os.path.exists') - @mock.patch('cloudinit.subp.subp') - def test_get_enabled_interfaces(self, mock_subp, mock_path_exists): - if is_notPi(): - return - # Simulate all interfaces enabled (spi, i2c, etc.) - mock_subp.side_effect = [("0", ""), ("0", ""), ("0", ""), ("0", "")] - config = { - RPI_INTERFACES_KEY: { - "spi": True, - "i2c": True, - "onewire": True, - "remote_gpio": True - } - } - handle("cc_rpi_interfaces", config, mock.Mock(), []) - # Assert all interface enabling commands were called - mock_subp.assert_any_call(['/usr/bin/raspi-config', - 'nonint', 'do_spi', '0']) - mock_subp.assert_any_call(['/usr/bin/raspi-config', - 'nonint', 'do_i2c', '0']) - mock_subp.assert_any_call(['/usr/bin/raspi-config', - 'nonint', 'do_onewire', '0']) - mock_subp.assert_any_call(['/usr/bin/raspi-config', - 'nonint', 'do_rgpio', '0']) -""" @skipUnlessJsonSchema() @@ -97,22 +20,34 @@ class TestCCRPiSchema: @pytest.mark.parametrize( "config, error_msg", [ - ({RPI_BASE_KEY: { RPI_INTERFACES_KEY: {"spi": True, "i2c": False}} }, None), + ( + { + RPI_BASE_KEY: { + RPI_INTERFACES_KEY: {"spi": True, "i2c": False} + } + }, + None, + ), ( {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"spi": "true"}}}, "'true' is not of type 'boolean'", ), ( - {RPI_BASE_KEY: { - RPI_INTERFACES_KEY: { - "serial": {"console": True, "hardware": False} + RPI_BASE_KEY: { + RPI_INTERFACES_KEY: { + "serial": {"console": True, "hardware": False} + } } - }}, + }, None, ), ( - {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": {"console": 123}}} }, + { + RPI_BASE_KEY: { + RPI_INTERFACES_KEY: {"serial": {"console": 123}} + } + }, "123 is not of type 'boolean'", ), ({RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}}, None), diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 9cd124506a3..a152a8437a7 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -107,10 +107,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): ("netbsd", ["netbsd"]), ("openbsd", ["openbsd"]), ("ubuntu", ["netplan", "eni", "sysconfig"]), - ( - "raspberry-pi-os", - ["netplan", "network-manager"] - ) + ("raspberry-pi-os", ["netplan", "network-manager"]), ), ) def test_variant_sets_network_renderer_priority_in_cloud_cfg( From 799c49742be23f7efe92b4cc0b303a1fd762c019 Mon Sep 17 00:00:00 2001 From: paulober Date: Wed, 9 Apr 2025 16:04:19 +0200 Subject: [PATCH 11/20] raspberry-pi-os: Added unit tests + requested changes Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 22 ++++- .../schemas/schema-cloud-config-v1.json | 7 +- cloudinit/distros/raspberry-pi-os.py | 44 ++------- pyproject.toml | 1 - .../unittests/config/test_cc_raspberry_pi.py | 50 +++++++++- .../unittests/distros/test_raspberry-pi-os.py | 93 +++++++++++++++++++ 6 files changed, 165 insertions(+), 52 deletions(-) create mode 100644 tests/unittests/distros/test_raspberry-pi-os.py diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index eee98ad87fa..deb4c1399ca 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -61,12 +61,25 @@ def is_pifive() -> bool: def configure_serial_interface(cfg: dict | bool, instCfg: Config) -> None: + def get_bool_field(name, default=False): + val = cfg.get(name, default) + if not isinstance(val, bool): + LOG.warning( + "Invalid value for %s.serial.%s: %s", + RPI_INTERFACES_KEY, + name, + val, + ) + return default + return val + enable_console = False enable_hw = False if isinstance(cfg, dict): - enable_console = cfg.get("console", False) - enable_hw = cfg.get("hardware", False) + enable_console = get_bool_field("console") + enable_hw = get_bool_field("hardware") + elif isinstance(cfg, bool): # default to enabling console as if < pi5 # this will also enable the hardware @@ -139,8 +152,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: for key in cfg[RPI_BASE_KEY]: if key == ENABLE_RPI_CONNECT_KEY: - # expect it to be a dictionary - enable = cfg[ENABLE_RPI_CONNECT_KEY] + enable = cfg[RPI_BASE_KEY][key] if isinstance(enable, bool): configure_rpi_connect(enable) @@ -169,7 +181,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: ) continue - enable = cfg[RPI_INTERFACES_KEY][subkey] + enable = cfg[RPI_BASE_KEY][key][subkey] if subkey == "serial": if not isinstance(enable, dict) and not isinstance( diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index 29497533e1a..b7074458806 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -2651,7 +2651,7 @@ "rpi": { "type": "object", "properties": { - "rpi_interfaces": { + "interfaces": { "type": "object", "properties": { "spi": { @@ -2697,11 +2697,6 @@ "type": "boolean", "description": "Enable remote GPIO interface. Default: ``false``.", "default": false - }, - "ssh": { - "type": "boolean", - "description": "Enable SSH. Default: ``false``.", - "default": false } } }, diff --git a/cloudinit/distros/raspberry-pi-os.py b/cloudinit/distros/raspberry-pi-os.py index 60164bcf14b..1dab647738e 100644 --- a/cloudinit/distros/raspberry-pi-os.py +++ b/cloudinit/distros/raspberry-pi-os.py @@ -1,19 +1,22 @@ -# Copyright (C) 2024 Raspberry Pi Ltd. All rights reserved. +# Copyright (C) 2024-2025 Raspberry Pi Ltd. All rights reserved. # # Author: Paul Oberosler # # This file is part of cloud-init. See LICENSE file for license information. -import os -import shutil +import logging from cloudinit import subp from cloudinit.distros import debian +LOG = logging.getLogger(__name__) + class Distro(debian.Distro): - def __init__(self, name, cfg, paths): - super().__init__(name, cfg, paths) + @classmethod + def get_logger(cls): + """For testing""" + return LOG def set_keymap(self, layout: str, model: str, variant: str, options: str): """Currently Raspberry Pi OS sys-mods only supports @@ -72,36 +75,7 @@ def add_user(self, name, **kwargs) -> bool: ) except subp.ProcessExecutionError as e: - self.log.error("Failed to setup user:", e) - return False - - # Alacarte fixes - try: - # Ensure the sudoers directory exists - os.makedirs( - f"/home/{name}/.local/share/applications", exist_ok=True - ) - os.makedirs( - f"/home/{name}/.local/share/desktop-directories", exist_ok=True - ) - - stat_info = os.stat(f"/home/{name}") - uid = stat_info.st_uid - gid = stat_info.st_gid - - paths = [ - f"/home/{name}/.local", - f"/home/{name}/.local/share", - f"/home/{name}/.local/share/applications", - f"/home/{name}/.local/share/desktop-directories", - ] - - for path in paths: - shutil.chown(path, user=uid, group=gid) - os.chmod(path, 0o755) - - except Exception as e: - self.log.error("Failed to setup userhome:", e) + LOG.error("Failed to setup user: %s", e) return False return True diff --git a/pyproject.toml b/pyproject.toml index 55d244691f3..efb8747ecb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,6 @@ module = [ "tests.unittests.config.test_cc_mcollective", "tests.unittests.config.test_cc_phone_home", "tests.unittests.config.test_cc_puppet", - "tests.unittests.config.test_cc_raspberry_pi", "tests.unittests.config.test_cc_resizefs", "tests.unittests.config.test_cc_resolv_conf", "tests.unittests.config.test_cc_rh_subscription", diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index 4d6d1346089..4fc2a1585e5 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -2,6 +2,7 @@ import pytest +import cloudinit.config.cc_raspberry_pi as cc_rpi from cloudinit.config.cc_raspberry_pi import ( ENABLE_RPI_CONNECT_KEY, RPI_BASE_KEY, @@ -12,11 +13,50 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.util import get_cloud + +M_PATH = "cloudinit.config.cc_raspberry_pi." + + +class TestHandleRaspberryPi: + @mock.patch(M_PATH + "configure_rpi_connect") + def test_handle_rpi_connect_enabled(self, m_connect): + cloud = get_cloud("raspberry-pi-os") + cfg = {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_connect.assert_called_once_with(True) + + @mock.patch(M_PATH + "configure_interface") + def test_handle_configure_interface_i2c(self, m_iface): + cloud = get_cloud("raspberry-pi-os") + cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"i2c": True}}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_iface.assert_called_once_with("i2c", True) + + @mock.patch(M_PATH + "configure_serial_interface") + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_handle_configure_serial_interface_dict(self, m_ispi5, m_serial): + cloud = get_cloud("raspberry-pi-os") + serial_value = { + "console": True, + "hardware": True, + } + cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": serial_value}}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_serial.assert_called_once_with(serial_value, cfg) + + @mock.patch(M_PATH + "configure_serial_interface") + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_handle_configure_serial_interface_bool(self, m_ispi5, m_serial): + cloud = get_cloud("raspberry-pi-os") + cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": True}}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_serial.assert_called_once_with(True, cfg) @skipUnlessJsonSchema() -class TestCCRPiSchema: +class TestRaspberryPiSchema: @pytest.mark.parametrize( "config, error_msg", [ @@ -30,7 +70,7 @@ class TestCCRPiSchema: ), ( {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"spi": "true"}}}, - "'true' is not of type 'boolean'", + f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.spi: 'true' is not of type 'boolean'", ), ( { @@ -48,12 +88,12 @@ class TestCCRPiSchema: RPI_INTERFACES_KEY: {"serial": {"console": 123}} } }, - "123 is not of type 'boolean'", + f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.serial.console: 123 is not of type 'boolean'", ), ({RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}}, None), ( {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: "true"}}, - "'true' is not of type 'boolean'", + f"{RPI_BASE_KEY}.{ENABLE_RPI_CONNECT_KEY}: 'true' is not of type 'boolean'", ), ], ) diff --git a/tests/unittests/distros/test_raspberry-pi-os.py b/tests/unittests/distros/test_raspberry-pi-os.py new file mode 100644 index 00000000000..2716c0b7215 --- /dev/null +++ b/tests/unittests/distros/test_raspberry-pi-os.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import fetch +from cloudinit.subp import ProcessExecutionError +from tests.unittests.helpers import CiTestCase, mock + +M_PATH = "cloudinit.distros.raspberry-pi-os." + + +class TestRaspberryPiOS(CiTestCase): + @mock.patch(M_PATH + "subp.subp") + def test_set_keymap_calls_imager_custom(self, m_subp): + cls = fetch("raspberry-pi-os") + distro = cls("raspberry-pi-os", {}, None) + distro.set_keymap("us", "pc105", "basic", "") + m_subp.assert_called_once_with( + ["/usr/lib/raspberrypi-sys-mods/imager_custom", "set_keymap", "us"] + ) + + @mock.patch(M_PATH + "subp.subp") + def test_apply_locale_happy_path(self, m_subp): + cls = fetch("raspberry-pi-os") + distro = cls("raspberry-pi-os", {}, None) + distro.apply_locale("en_GB.UTF-8") + m_subp.assert_called_once_with( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + "en_GB.UTF-8", + ] + ) + + @mock.patch(M_PATH + "subp.subp") + def test_apply_locale_fallback_to_utf8(self, m_subp): + m_subp.side_effect = [ + ProcessExecutionError("Invalid locale"), # Simulate failure + None, # Fallback succeeds + ] + cls = fetch("raspberry-pi-os") + distro = cls("raspberry-pi-os", {}, None) + distro.apply_locale("en_GB") + assert m_subp.call_count == 2 + m_subp.assert_any_call( + ["/usr/bin/raspi-config", "nonint", "do_change_locale", "en_GB"] + ) + m_subp.assert_any_call( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + "en_GB.UTF-8", + ] + ) + + @mock.patch(M_PATH + "subp.subp") + def test_add_user_happy_path(self, m_subp): + cls = fetch("raspberry-pi-os") + distro = cls("raspberry-pi-os", {}, None) + # Mock the superclass add_user to return True + with mock.patch( + "cloudinit.distros.debian.Distro.add_user", return_value=True + ): + assert distro.add_user("pi") is True + m_subp.assert_called_once_with( + ["/usr/bin/rename-user", "-f", "-s"], + update_env={"SUDO_USER": "pi"}, + ) + + @mock.patch(M_PATH + "subp.subp") + def test_add_user_existing_user(self, m_subp): + cls = fetch("raspberry-pi-os") + distro = cls("raspberry-pi-os", {}, None) + with mock.patch( + "cloudinit.distros.debian.Distro.add_user", return_value=False + ): + assert distro.add_user("pi") is False + m_subp.assert_not_called() + + @mock.patch( + M_PATH + "subp.subp", + side_effect=ProcessExecutionError("rename-user failed"), + ) + def test_add_user_rename_fails_logs_error(self, m_subp): + cls = fetch("raspberry-pi-os") + distro = cls("raspberry-pi-os", {}, None) + + with mock.patch( + "cloudinit.distros.debian.Distro.add_user", return_value=True + ): + with self.assertLogs(cls.get_logger().name, level="ERROR") as cm: + assert distro.add_user("pi") is False + assert "Failed to setup user" in cm.output[0] From 502088d31fd4c862c22ef0128424a53087000856 Mon Sep 17 00:00:00 2001 From: paulober Date: Fri, 11 Apr 2025 00:16:36 +0200 Subject: [PATCH 12/20] Using caplog + more logging Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 17 ++++++------- cloudinit/distros/raspberry-pi-os.py | 25 +++++++++---------- .../unittests/distros/test_raspberry-pi-os.py | 21 +++++++++------- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index deb4c1399ca..fa708f6b54b 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -12,6 +12,8 @@ from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE +from typing import Union + LOG = logging.getLogger(__name__) RPI_BASE_KEY = "rpi" RPI_INTERFACES_KEY = "interfaces" @@ -45,7 +47,6 @@ def configure_rpi_connect(enable: bool) -> None: LOG.error("Failed to configure rpi-connect: %s", e) -# TODO: test def require_reboot(cfg: Config) -> None: cfg["power_state"] = cfg.get("power_state", {}) cfg["power_state"]["mode"] = cfg["power_state"].get("mode", "reboot") @@ -60,9 +61,9 @@ def is_pifive() -> bool: return False -def configure_serial_interface(cfg: dict | bool, instCfg: Config) -> None: - def get_bool_field(name, default=False): - val = cfg.get(name, default) +def configure_serial_interface(cfg: Union[dict, bool], instCfg: Config) -> None: + def get_bool_field(cfg_dict: dict, name: str, default=False): + val = cfg_dict.get(name, default) if not isinstance(val, bool): LOG.warning( "Invalid value for %s.serial.%s: %s", @@ -77,8 +78,8 @@ def get_bool_field(name, default=False): enable_hw = False if isinstance(cfg, dict): - enable_console = get_bool_field("console") - enable_hw = get_bool_field("hardware") + enable_console = get_bool_field(cfg, "console") + enable_hw = get_bool_field(cfg, "hardware") elif isinstance(cfg, bool): # default to enabling console as if < pi5 @@ -184,9 +185,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: enable = cfg[RPI_BASE_KEY][key][subkey] if subkey == "serial": - if not isinstance(enable, dict) and not isinstance( - enable, bool - ): + if not isinstance(enable, (dict, bool)): LOG.warning( "Invalid value for %s.%s: %s", RPI_INTERFACES_KEY, diff --git a/cloudinit/distros/raspberry-pi-os.py b/cloudinit/distros/raspberry-pi-os.py index 1dab647738e..f1bd111acc4 100644 --- a/cloudinit/distros/raspberry-pi-os.py +++ b/cloudinit/distros/raspberry-pi-os.py @@ -13,11 +13,6 @@ class Distro(debian.Distro): - @classmethod - def get_logger(cls): - """For testing""" - return LOG - def set_keymap(self, layout: str, model: str, variant: str, options: str): """Currently Raspberry Pi OS sys-mods only supports setting the layout""" @@ -41,14 +36,18 @@ def apply_locale(self, locale, out_fn=None, keyname="LANG"): ] ) except subp.ProcessExecutionError: - subp.subp( - [ - "/usr/bin/raspi-config", - "nonint", - "do_change_locale", - f"{locale}.UTF-8", - ] - ) + if not locale.endswith(".UTF-8"): + LOG.info("Trying to set locale %s.UTF-8", locale) + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + f"{locale}.UTF-8", + ] + ) + else: + LOG.error("Failed to set locale %s") def add_user(self, name, **kwargs) -> bool: """ diff --git a/tests/unittests/distros/test_raspberry-pi-os.py b/tests/unittests/distros/test_raspberry-pi-os.py index 2716c0b7215..401961074c3 100644 --- a/tests/unittests/distros/test_raspberry-pi-os.py +++ b/tests/unittests/distros/test_raspberry-pi-os.py @@ -1,13 +1,15 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging + from cloudinit.distros import fetch from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import mock M_PATH = "cloudinit.distros.raspberry-pi-os." -class TestRaspberryPiOS(CiTestCase): +class TestRaspberryPiOS: @mock.patch(M_PATH + "subp.subp") def test_set_keymap_calls_imager_custom(self, m_subp): cls = fetch("raspberry-pi-os") @@ -81,13 +83,14 @@ def test_add_user_existing_user(self, m_subp): M_PATH + "subp.subp", side_effect=ProcessExecutionError("rename-user failed"), ) - def test_add_user_rename_fails_logs_error(self, m_subp): + @mock.patch( + "cloudinit.distros.debian.Distro.add_user", + return_value=True + ) + def test_add_user_rename_fails_logs_error(self, m_super_add_user, m_subp, caplog): cls = fetch("raspberry-pi-os") distro = cls("raspberry-pi-os", {}, None) - with mock.patch( - "cloudinit.distros.debian.Distro.add_user", return_value=True - ): - with self.assertLogs(cls.get_logger().name, level="ERROR") as cm: - assert distro.add_user("pi") is False - assert "Failed to setup user" in cm.output[0] + with caplog.at_level(logging.ERROR): + assert distro.add_user("pi") is False + assert "Failed to setup user" in caplog.text From be0675180a264cf95cefebaa7547cc07b670133a Mon Sep 17 00:00:00 2001 From: paulober Date: Fri, 11 Apr 2025 00:36:19 +0200 Subject: [PATCH 13/20] Formatting Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 7 ++++--- tests/unittests/config/test_cc_raspberry_pi.py | 9 ++++++--- tests/unittests/distros/test_raspberry-pi-os.py | 9 ++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index fa708f6b54b..68c6b057ba7 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +from typing import Union from cloudinit import subp from cloudinit.cloud import Cloud @@ -12,8 +13,6 @@ from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE -from typing import Union - LOG = logging.getLogger(__name__) RPI_BASE_KEY = "rpi" RPI_INTERFACES_KEY = "interfaces" @@ -61,7 +60,9 @@ def is_pifive() -> bool: return False -def configure_serial_interface(cfg: Union[dict, bool], instCfg: Config) -> None: +def configure_serial_interface( + cfg: Union[dict, bool], instCfg: Config +) -> None: def get_bool_field(cfg_dict: dict, name: str, default=False): val = cfg_dict.get(name, default) if not isinstance(val, bool): diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index 4fc2a1585e5..52923a02d1c 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -70,7 +70,8 @@ class TestRaspberryPiSchema: ), ( {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"spi": "true"}}}, - f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.spi: 'true' is not of type 'boolean'", + f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.spi: 'true'" + " is not of type 'boolean'", ), ( { @@ -88,12 +89,14 @@ class TestRaspberryPiSchema: RPI_INTERFACES_KEY: {"serial": {"console": 123}} } }, - f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.serial.console: 123 is not of type 'boolean'", + f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.serial.console: " + "123 is not of type 'boolean'", ), ({RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}}, None), ( {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: "true"}}, - f"{RPI_BASE_KEY}.{ENABLE_RPI_CONNECT_KEY}: 'true' is not of type 'boolean'", + f"{RPI_BASE_KEY}.{ENABLE_RPI_CONNECT_KEY}: 'true'" + " is not of type 'boolean'", ), ], ) diff --git a/tests/unittests/distros/test_raspberry-pi-os.py b/tests/unittests/distros/test_raspberry-pi-os.py index 401961074c3..d9f6f08efa4 100644 --- a/tests/unittests/distros/test_raspberry-pi-os.py +++ b/tests/unittests/distros/test_raspberry-pi-os.py @@ -83,11 +83,10 @@ def test_add_user_existing_user(self, m_subp): M_PATH + "subp.subp", side_effect=ProcessExecutionError("rename-user failed"), ) - @mock.patch( - "cloudinit.distros.debian.Distro.add_user", - return_value=True - ) - def test_add_user_rename_fails_logs_error(self, m_super_add_user, m_subp, caplog): + @mock.patch("cloudinit.distros.debian.Distro.add_user", return_value=True) + def test_add_user_rename_fails_logs_error( + self, m_super_add_user, m_subp, caplog + ): cls = fetch("raspberry-pi-os") distro = cls("raspberry-pi-os", {}, None) From ee7975f1f7e67e78ecd60fbe205d241dec9ec305 Mon Sep 17 00:00:00 2001 From: paulober Date: Sat, 12 Apr 2025 22:28:41 +0200 Subject: [PATCH 14/20] Fix module naming Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- .../{raspberry-pi-os.py => raspberry_pi_os.py} | 0 tests/unittests/config/test_cc_netplan_nm_patch.py | 9 --------- ..._raspberry-pi-os.py => test_raspberry_pi_os.py} | 14 +++++++------- 3 files changed, 7 insertions(+), 16 deletions(-) rename cloudinit/distros/{raspberry-pi-os.py => raspberry_pi_os.py} (100%) delete mode 100644 tests/unittests/config/test_cc_netplan_nm_patch.py rename tests/unittests/distros/{test_raspberry-pi-os.py => test_raspberry_pi_os.py} (91%) diff --git a/cloudinit/distros/raspberry-pi-os.py b/cloudinit/distros/raspberry_pi_os.py similarity index 100% rename from cloudinit/distros/raspberry-pi-os.py rename to cloudinit/distros/raspberry_pi_os.py diff --git a/tests/unittests/config/test_cc_netplan_nm_patch.py b/tests/unittests/config/test_cc_netplan_nm_patch.py deleted file mode 100644 index 6f76cba7446..00000000000 --- a/tests/unittests/config/test_cc_netplan_nm_patch.py +++ /dev/null @@ -1,9 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from tests.unittests.helpers import skipUnlessJsonSchema - - -@skipUnlessJsonSchema() -class TestCCNetplanNmPatch: - def test_schema_validation(self): - pass diff --git a/tests/unittests/distros/test_raspberry-pi-os.py b/tests/unittests/distros/test_raspberry_pi_os.py similarity index 91% rename from tests/unittests/distros/test_raspberry-pi-os.py rename to tests/unittests/distros/test_raspberry_pi_os.py index d9f6f08efa4..508b84bba11 100644 --- a/tests/unittests/distros/test_raspberry-pi-os.py +++ b/tests/unittests/distros/test_raspberry_pi_os.py @@ -6,13 +6,13 @@ from cloudinit.subp import ProcessExecutionError from tests.unittests.helpers import mock -M_PATH = "cloudinit.distros.raspberry-pi-os." +M_PATH = "cloudinit.distros.raspberry_pi_os." class TestRaspberryPiOS: @mock.patch(M_PATH + "subp.subp") def test_set_keymap_calls_imager_custom(self, m_subp): - cls = fetch("raspberry-pi-os") + cls = fetch("raspberry_pi_os") distro = cls("raspberry-pi-os", {}, None) distro.set_keymap("us", "pc105", "basic", "") m_subp.assert_called_once_with( @@ -21,7 +21,7 @@ def test_set_keymap_calls_imager_custom(self, m_subp): @mock.patch(M_PATH + "subp.subp") def test_apply_locale_happy_path(self, m_subp): - cls = fetch("raspberry-pi-os") + cls = fetch("raspberry_pi_os") distro = cls("raspberry-pi-os", {}, None) distro.apply_locale("en_GB.UTF-8") m_subp.assert_called_once_with( @@ -39,7 +39,7 @@ def test_apply_locale_fallback_to_utf8(self, m_subp): ProcessExecutionError("Invalid locale"), # Simulate failure None, # Fallback succeeds ] - cls = fetch("raspberry-pi-os") + cls = fetch("raspberry_pi_os") distro = cls("raspberry-pi-os", {}, None) distro.apply_locale("en_GB") assert m_subp.call_count == 2 @@ -57,7 +57,7 @@ def test_apply_locale_fallback_to_utf8(self, m_subp): @mock.patch(M_PATH + "subp.subp") def test_add_user_happy_path(self, m_subp): - cls = fetch("raspberry-pi-os") + cls = fetch("raspberry_pi_os") distro = cls("raspberry-pi-os", {}, None) # Mock the superclass add_user to return True with mock.patch( @@ -71,7 +71,7 @@ def test_add_user_happy_path(self, m_subp): @mock.patch(M_PATH + "subp.subp") def test_add_user_existing_user(self, m_subp): - cls = fetch("raspberry-pi-os") + cls = fetch("raspberry_pi_os") distro = cls("raspberry-pi-os", {}, None) with mock.patch( "cloudinit.distros.debian.Distro.add_user", return_value=False @@ -87,7 +87,7 @@ def test_add_user_existing_user(self, m_subp): def test_add_user_rename_fails_logs_error( self, m_super_add_user, m_subp, caplog ): - cls = fetch("raspberry-pi-os") + cls = fetch("raspberry_pi_os") distro = cls("raspberry-pi-os", {}, None) with caplog.at_level(logging.ERROR): From 3b26d7374c814dd01723b8699094af2667a32128 Mon Sep 17 00:00:00 2001 From: paulober Date: Mon, 21 Apr 2025 13:21:43 +0200 Subject: [PATCH 15/20] Fix reboot request and recommendations Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 32 +++++++++++++------ .../schemas/schema-cloud-config-v1.json | 6 ++-- cloudinit/distros/__init__.py | 5 +++ pyproject.toml | 5 --- .../unittests/config/test_cc_raspberry_pi.py | 12 +++---- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index 68c6b057ba7..c4665853b51 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -5,6 +5,8 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +import os +import subprocess from typing import Union from cloudinit import subp @@ -46,12 +48,6 @@ def configure_rpi_connect(enable: bool) -> None: LOG.error("Failed to configure rpi-connect: %s", e) -def require_reboot(cfg: Config) -> None: - cfg["power_state"] = cfg.get("power_state", {}) - cfg["power_state"]["mode"] = cfg["power_state"].get("mode", "reboot") - cfg["power_state"]["condition"] = True - - def is_pifive() -> bool: try: subp.subp(["/usr/bin/raspi-config", "nonint", "is_pifive"]) @@ -61,7 +57,7 @@ def is_pifive() -> bool: def configure_serial_interface( - cfg: Union[dict, bool], instCfg: Config + cfg: Union[dict, bool], instCfg: Config, cloud: Cloud ) -> None: def get_bool_field(cfg_dict: dict, name: str, default=False): val = cfg_dict.get(name, default) @@ -115,9 +111,20 @@ def get_bool_field(cfg_dict: dict, name: str, default=False): except subp.ProcessExecutionError as e: LOG.error("Failed to configure serial hardware: %s", e) - require_reboot(instCfg) + # Reboot to apply changes + cmd = cloud.distro.shutdown_command( + mode="reboot", + delay="now", + message="Rebooting to apply serial console changes", + ) + proc = subprocess.Popen( + cmd, stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT + ) + ret = proc.returncode except subp.ProcessExecutionError as e: LOG.error("Failed to configure serial console: %s", e) + os._exit(254) + os._exit(ret) def configure_interface(iface: str, enable: bool) -> None: @@ -175,8 +182,13 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: LOG.debug("Empty value for %s. Skipping...", key) return + subkeys = list(cfg[RPI_BASE_KEY][key].keys()) + # Move " serial" to the end if it exists + if "serial" in subkeys: + subkeys.append(subkeys.pop(subkeys.index("serial"))) + # check for supported ARM interfaces - for subkey in cfg[RPI_BASE_KEY][key]: + for subkey in subkeys: if subkey not in SUPPORTED_INTERFACES.keys(): LOG.warning( "Invalid key for %s: %s", RPI_INTERFACES_KEY, subkey @@ -194,7 +206,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: enable, ) else: - configure_serial_interface(enable, cfg) + configure_serial_interface(enable, cfg, cloud) continue if isinstance(enable, bool): diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index b7074458806..941eac3e9dc 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -3964,6 +3964,9 @@ { "$ref": "#/$defs/cc_puppet" }, + { + "$ref": "#/$defs/cc_raspberry_pi" + }, { "$ref": "#/$defs/cc_resizefs" }, @@ -3973,9 +3976,6 @@ { "$ref": "#/$defs/cc_rh_subscription" }, - { - "$ref": "#/$defs/cc_raspberry_pi" - }, { "$ref": "#/$defs/cc_rsyslog" }, diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index c11aeb79d5e..eb58095cb71 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -1768,6 +1768,11 @@ def _get_arch_package_mirror_info(package_mirrors, arch): def fetch(name: str) -> Type[Distro]: locs, looked_locs = importer.find_module(name, ["", __name__], ["Distro"]) + if not locs: + # Some distros may have a `-` in the name but an `_` in the module + locs, _ = importer.find_module( + name.replace("-", "_"), ["", __name__], ["Distro"] + ) if not locs: raise ImportError( "No distribution found for distro %s (searched %s)" diff --git a/pyproject.toml b/pyproject.toml index efb8747ecb9..faf09c234f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,11 +132,6 @@ module = [ "tests.unittests.config.test_cc_resizefs", "tests.unittests.config.test_cc_resolv_conf", "tests.unittests.config.test_cc_rh_subscription", - "tests.unittests.config.test_cc_rsyslog", - "tests.unittests.config.test_cc_runcmd", - "tests.unittests.config.test_cc_snap", - "tests.unittests.config.test_cc_ssh", - "tests.unittests.config.test_cc_timezone", "tests.unittests.config.test_cc_ubuntu_autoinstall", "tests.unittests.config.test_cc_update_etc_hosts", "tests.unittests.config.test_cc_users_groups", diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index 52923a02d1c..c82e50a4e1d 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -22,14 +22,14 @@ class TestHandleRaspberryPi: @mock.patch(M_PATH + "configure_rpi_connect") def test_handle_rpi_connect_enabled(self, m_connect): - cloud = get_cloud("raspberry-pi-os") + cloud = get_cloud("raspberry_pi_os") cfg = {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}} cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) m_connect.assert_called_once_with(True) @mock.patch(M_PATH + "configure_interface") def test_handle_configure_interface_i2c(self, m_iface): - cloud = get_cloud("raspberry-pi-os") + cloud = get_cloud("raspberry_pi_os") cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"i2c": True}}} cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) m_iface.assert_called_once_with("i2c", True) @@ -37,22 +37,22 @@ def test_handle_configure_interface_i2c(self, m_iface): @mock.patch(M_PATH + "configure_serial_interface") @mock.patch(M_PATH + "is_pifive", return_value=True) def test_handle_configure_serial_interface_dict(self, m_ispi5, m_serial): - cloud = get_cloud("raspberry-pi-os") + cloud = get_cloud("raspberry_pi_os") serial_value = { "console": True, "hardware": True, } cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": serial_value}}} cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) - m_serial.assert_called_once_with(serial_value, cfg) + m_serial.assert_called_once_with(serial_value, cfg, cloud) @mock.patch(M_PATH + "configure_serial_interface") @mock.patch(M_PATH + "is_pifive", return_value=True) def test_handle_configure_serial_interface_bool(self, m_ispi5, m_serial): - cloud = get_cloud("raspberry-pi-os") + cloud = get_cloud("raspberry_pi_os") cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": True}}} cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) - m_serial.assert_called_once_with(True, cfg) + m_serial.assert_called_once_with(True, cfg, cloud) @skipUnlessJsonSchema() From a9a8997f275f4f11fac6b69b0ee12c6046d5138e Mon Sep 17 00:00:00 2001 From: paulober Date: Mon, 21 Apr 2025 15:02:32 +0200 Subject: [PATCH 16/20] Add more tests Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- .../unittests/config/test_cc_raspberry_pi.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index c82e50a4e1d..8abe9ed8e75 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -13,6 +13,7 @@ get_schema, validate_cloudconfig_schema, ) +from cloudinit.subp import ProcessExecutionError from tests.unittests.helpers import mock, skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -55,6 +56,88 @@ def test_handle_configure_serial_interface_bool(self, m_ispi5, m_serial): m_serial.assert_called_once_with(True, cfg, cloud) +class TestRaspberryPiMethods: + @mock.patch("cloudinit.subp.subp") + def test_configure_rpi_connect_enable(self, m_subp): + cc_rpi.configure_rpi_connect(True) + m_subp.assert_called_once_with( + ["/usr/bin/raspi-config", "do_rpi_connect", "0"] + ) + + @mock.patch( + "cloudinit.subp.subp", + side_effect=ProcessExecutionError("1", [], "fail"), + ) + def test_configure_rpi_connect_failure(self, m_subp): + cc_rpi.configure_rpi_connect(False) # Should log error but not raise + + @mock.patch("cloudinit.subp.subp", return_value=("ok", "")) + def test_is_pifive_true(self, m_subp): + assert cc_rpi.is_pifive() is True + + @mock.patch( + "cloudinit.subp.subp", + side_effect=ProcessExecutionError("1", [], "fail"), + ) + def test_is_pifive_false(self, m_subp): + assert cc_rpi.is_pifive() is False + + @mock.patch("cloudinit.subp.subp") + def test_configure_interface_valid(self, m_subp): + cc_rpi.configure_interface("i2c", True) + m_subp.assert_called_once_with( + ["/usr/bin/raspi-config", "nonint", "do_i2c", "0"] + ) + + def test_configure_interface_invalid(self): + with pytest.raises(AssertionError): + cc_rpi.configure_interface("invalid_iface", True) + + @mock.patch("cloudinit.subp.subp") + @mock.patch("subprocess.Popen") + @mock.patch(M_PATH + "is_pifive", return_value=False) + def test_configure_serial_interface_dict( + self, m_is_pifive, m_popen, m_subp + ): + cloud = get_cloud("raspberry_pi_os") + m_popen.return_value.returncode = 0 + cfg = {"console": True, "hardware": False} + with mock.patch("os._exit", side_effect=SystemExit) as m_exit: + with pytest.raises(SystemExit): + cc_rpi.configure_serial_interface(cfg, None, cloud) + m_subp.assert_any_call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_CONS_FN, + "0", + ] + ) + m_subp.assert_any_call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_HW_FN, + "0", + ] + ) + m_exit.assert_called_once_with(0) + + @mock.patch("cloudinit.subp.subp") + @mock.patch("subprocess.Popen") + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_configure_serial_interface_bool( + self, m_is_pifive, m_popen, m_subp + ): + cloud = get_cloud("raspberry_pi_os") + m_popen.return_value.returncode = 0 + with mock.patch("os._exit", side_effect=SystemExit) as m_exit: + with pytest.raises(SystemExit): + cc_rpi.configure_serial_interface(True, None, cloud) + assert m_subp.call_count == 2 + m_exit.assert_called_once_with(0) + + @skipUnlessJsonSchema() class TestRaspberryPiSchema: @pytest.mark.parametrize( From 9bbe41b82d7ab70248c314d520eb0d57da5a97cb Mon Sep 17 00:00:00 2001 From: paulober Date: Tue, 29 Apr 2025 00:50:17 +0200 Subject: [PATCH 17/20] Remove custom exit Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index c4665853b51..45d28b38f5c 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -117,14 +117,11 @@ def get_bool_field(cfg_dict: dict, name: str, default=False): delay="now", message="Rebooting to apply serial console changes", ) - proc = subprocess.Popen( + _ = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT ) - ret = proc.returncode except subp.ProcessExecutionError as e: LOG.error("Failed to configure serial console: %s", e) - os._exit(254) - os._exit(ret) def configure_interface(iface: str, enable: bool) -> None: From 1d287e5b3d4b20e21d5b3735bb8602cf26af50ee Mon Sep 17 00:00:00 2001 From: paulober Date: Wed, 30 Apr 2025 09:25:53 +0200 Subject: [PATCH 18/20] Fix reboot cmd exec Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index 45d28b38f5c..438ac14cddf 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -117,9 +117,7 @@ def get_bool_field(cfg_dict: dict, name: str, default=False): delay="now", message="Rebooting to apply serial console changes", ) - _ = subprocess.Popen( - cmd, stdin=subprocess.PIPE, stdout=None, stderr=subprocess.STDOUT - ) + subp.subp(cmd) except subp.ProcessExecutionError as e: LOG.error("Failed to configure serial console: %s", e) From 3387cb2f3fedfd3374256bd12b527bf7e4f8656c Mon Sep 17 00:00:00 2001 From: paulober Date: Fri, 2 May 2025 14:13:56 +0200 Subject: [PATCH 19/20] Fix tests Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- cloudinit/config/cc_raspberry_pi.py | 2 - .../unittests/config/test_cc_raspberry_pi.py | 96 ++++++++++++------- 2 files changed, 60 insertions(+), 38 deletions(-) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py index 438ac14cddf..a80a8f08b41 100644 --- a/cloudinit/config/cc_raspberry_pi.py +++ b/cloudinit/config/cc_raspberry_pi.py @@ -5,8 +5,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging -import os -import subprocess from typing import Union from cloudinit import subp diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py index 8abe9ed8e75..c0bfc8591cf 100644 --- a/tests/unittests/config/test_cc_raspberry_pi.py +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -94,48 +94,72 @@ def test_configure_interface_invalid(self): cc_rpi.configure_interface("invalid_iface", True) @mock.patch("cloudinit.subp.subp") - @mock.patch("subprocess.Popen") - @mock.patch(M_PATH + "is_pifive", return_value=False) - def test_configure_serial_interface_dict( - self, m_is_pifive, m_popen, m_subp - ): + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_configure_serial_interface_dict_config(self, m_ispi5, m_subp): cloud = get_cloud("raspberry_pi_os") - m_popen.return_value.returncode = 0 cfg = {"console": True, "hardware": False} - with mock.patch("os._exit", side_effect=SystemExit) as m_exit: - with pytest.raises(SystemExit): - cc_rpi.configure_serial_interface(cfg, None, cloud) - m_subp.assert_any_call( - [ - "/usr/bin/raspi-config", - "nonint", - cc_rpi.RASPI_CONFIG_SERIAL_CONS_FN, - "0", - ] - ) - m_subp.assert_any_call( - [ - "/usr/bin/raspi-config", - "nonint", - cc_rpi.RASPI_CONFIG_SERIAL_HW_FN, - "0", - ] - ) - m_exit.assert_called_once_with(0) + + # Simulate is_pifive returning True to prevent enable_hw override + with mock.patch.object( + cloud.distro, "shutdown_command", return_value=["reboot"] + ): + cc_rpi.configure_serial_interface(cfg, {}, cloud) + + expected_calls = [ + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_CONS_FN, + "0", + ] + ), + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_HW_FN, + "1", + ] + ), + mock.call(["reboot"]), + ] + m_subp.assert_has_calls(expected_calls, any_order=False) @mock.patch("cloudinit.subp.subp") - @mock.patch("subprocess.Popen") - @mock.patch(M_PATH + "is_pifive", return_value=True) - def test_configure_serial_interface_bool( - self, m_is_pifive, m_popen, m_subp + @mock.patch(M_PATH + "is_pifive", return_value=False) + def test_configure_serial_interface_boolean_config_non_pi5( + self, m_ispi5, m_subp ): cloud = get_cloud("raspberry_pi_os") - m_popen.return_value.returncode = 0 - with mock.patch("os._exit", side_effect=SystemExit) as m_exit: - with pytest.raises(SystemExit): - cc_rpi.configure_serial_interface(True, None, cloud) - assert m_subp.call_count == 2 - m_exit.assert_called_once_with(0) + + with mock.patch.object( + cloud.distro, + "shutdown_command", + return_value=["shutdown", "-r", "now"], + ): + cc_rpi.configure_serial_interface(True, {}, cloud) + + expected_calls = [ + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_CONS_FN, + "0", + ] + ), + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_HW_FN, + "0", + ] + ), + mock.call(["shutdown", "-r", "now"]), + ] + m_subp.assert_has_calls(expected_calls, any_order=False) @skipUnlessJsonSchema() From c8a0828b38d67bdd7eadd669d766c58af5e86b25 Mon Sep 17 00:00:00 2001 From: paulober Date: Thu, 8 May 2025 16:02:59 +0200 Subject: [PATCH 20/20] Update examples Signed-off-by: paulober <44974737+paulober@users.noreply.github.com> --- doc/module-docs/cc_raspberry_pi/example2.yaml | 5 +++-- doc/module-docs/cc_raspberry_pi/example3.yaml | 11 ++++++----- doc/module-docs/cc_raspberry_pi/example4.yaml | 15 ++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/doc/module-docs/cc_raspberry_pi/example2.yaml b/doc/module-docs/cc_raspberry_pi/example2.yaml index eeae5fd71c4..83ef6390b7a 100644 --- a/doc/module-docs/cc_raspberry_pi/example2.yaml +++ b/doc/module-docs/cc_raspberry_pi/example2.yaml @@ -1,3 +1,4 @@ #cloud-config -rpi_interfaces: - serial: true +rpi: + interfaces: + serial: true diff --git a/doc/module-docs/cc_raspberry_pi/example3.yaml b/doc/module-docs/cc_raspberry_pi/example3.yaml index b7eb744c243..eb74656aa7f 100644 --- a/doc/module-docs/cc_raspberry_pi/example3.yaml +++ b/doc/module-docs/cc_raspberry_pi/example3.yaml @@ -1,6 +1,7 @@ #cloud-config -rpi_interfaces: - serial: - # Pi 5 only | disabling hardware while enabling console - console: true - hardware: false +rpi: + interfaces: + serial: + # Pi 5 only | disabling hardware while enabling console + console: true + hardware: false diff --git a/doc/module-docs/cc_raspberry_pi/example4.yaml b/doc/module-docs/cc_raspberry_pi/example4.yaml index 4822fd4ae70..c600a276ffc 100644 --- a/doc/module-docs/cc_raspberry_pi/example4.yaml +++ b/doc/module-docs/cc_raspberry_pi/example4.yaml @@ -1,8 +1,9 @@ #cloud-config -rpi_interfaces: - ssh: true - # works on all Pi models - # only enables the UART hardware without binding it to the console - serial: - console: false - hardware: true +rpi: + interfaces: + ssh: true + # works on all Pi models + # only enables the UART hardware without binding it to the console + serial: + console: false + hardware: true