From 38bb84dd674805124021b5b7c3cf1c01f3f60915 Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Tue, 1 Apr 2025 11:38:03 +0200 Subject: [PATCH 01/12] fix: hetzner networking setup (WIP) --- cloudinit/sources/DataSourceHetzner.py | 103 ++++++++++++++++++++++++- cloudinit/sources/helpers/hetzner.py | 2 +- 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 7b919f66fa4..13da635e516 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -19,6 +19,8 @@ BUILTIN_DS_CONFIG = { "metadata_url": BASE_URL_V1 + "/metadata", + "metadata_private_networks_url": BASE_URL_V1 + + "/metadata/private-networks", "userdata_url": BASE_URL_V1 + "/userdata", } @@ -26,6 +28,13 @@ MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 +# Do not re-configure the network on non-Hetzner network interface +# changes. Currently, Hetzner private network addresses start with 0x86. +EXTRA_HOTPLUG_UDEV_RULES = """ +SUBSYSTEM=="net", ATTR{address}=="86:*", GOTO="cloudinit_hook" +GOTO="cloudinit_end" +""" + class DataSourceHetzner(sources.DataSource): @@ -42,6 +51,9 @@ def __init__(self, sys_cfg, distro, paths): ] ) self.metadata_address = self.ds_cfg["metadata_url"] + self.metadata_private_networks_address = self.ds_cfg[ + "metadata_private_networks_url" + ] self.userdata_address = self.ds_cfg["userdata_url"] self.retries = self.ds_cfg.get("retries", MD_RETRIES) self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT) @@ -50,6 +62,8 @@ def __init__(self, sys_cfg, distro, paths): self.dsmode = sources.DSMODE_NETWORK self.metadata_full = None + self.extra_hotplug_udev_rules = EXTRA_HOTPLUG_UDEV_RULES + def _get_data(self): (on_hetzner, serial) = get_hcloud_data() @@ -72,6 +86,12 @@ def _get_data(self): sec_between=self.wait_retry, retries=self.retries, ) + md["private-networks"] = hc_helper.read_metadata( + self.metadata_private_networks_address, + timeout=self.timeout, + sec_between=self.wait_retry, + retries=self.retries, + ) ud = hc_helper.read_userdata( self.userdata_address, timeout=self.timeout, @@ -99,6 +119,7 @@ def _get_data(self): self.metadata["local-hostname"] = md["hostname"] self.metadata["network-config"] = md.get("network-config", None) self.metadata["public-keys"] = md.get("public-keys", None) + self.metadata["private-networks"] = md.get("private-networks", []) self.vendordata_raw = md.get("vendor_data", None) # instance-id and serial from SMBIOS should be identical @@ -135,9 +156,87 @@ def network_config(self): if not _net_config: raise RuntimeError("Unable to get meta-data from server....") - self._network_config = _net_config + self._network_config_v2 = { + "version": 2, + "ethernets": {}, + } + self._network_config_v2["ethernets"] = self._network_config_v1_to_v2( + _net_config + ) - return self._network_config + for private_network in self.metadata.get("private-networks", []): + network_config_v2 = { + "match": { + "macaddress": private_network["mac_address"], + }, + "dhcp4": True, + } + idx = private_network["interface_num"] + # The key name (priv...) is just a virtual interface name. + # To rename the interface, "set-name" must be used, but we + # want to keep the OS-chosen name. + self._network_config_v2["ethernets"][ + f"priv{idx}" + ] = network_config_v2 + + return self._network_config_v2 + + def _network_config_v1_to_v2(self, network_config_v1): + ethernets = {} + + for network in network_config_v1["config"]: + networkv2 = { + "match": { + "macaddress": network["mac_address"], + }, + "set-name": network["name"], + "addresses": [], + "nameservers": { + "addresses": [], + }, + "routes": [], + } + + for subnet in network["subnets"]: + if ( + "ipv4" in subnet + and subnet["ipv4"] + and subnet["type"] == "dhcp" + ): + networkv2["dhcp4"] = True + if ( + "ipv6" in subnet + and subnet["ipv6"] + and subnet["type"] == "dhcp" + ): + networkv2["dhcp6"] = True + + if subnet["type"] == "static": + if "address" in subnet: + networkv2["addresses"].append(subnet["address"]) + + if "dns_nameservers" in subnet: + for ns in subnet["dns_nameservers"]: + networkv2["nameservers"]["addresses"].append(ns) + + target = None + if "ipv4" in subnet and subnet["ipv4"]: + target = "0.0.0.0/0" + elif "ipv6" in subnet and subnet["ipv6"]: + target = "::/0" + + if "gateway" in subnet and target: + networkv2["routes"].append( + { + "on-link": True, + "to": target, + "via": subnet["gateway"], + } + ) + + ethernets[network["name"]] = networkv2 + + return ethernets def get_hcloud_data(): diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index a1fd92ff37d..50fbcb0468a 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -12,7 +12,7 @@ def read_metadata(url, timeout=2, sec_between=2, retries=30): ) if not response.ok(): raise RuntimeError("unable to read metadata at %s" % url) - return util.load_yaml(response.contents.decode()) + return util.load_yaml(response.contents.decode(), allowed=(dict, list)) def read_userdata(url, timeout=2, sec_between=2, retries=30): From 0149efbee5d4022e3911cf76228d6a7709af1dcf Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Mon, 18 Aug 2025 13:24:38 +0000 Subject: [PATCH 02/12] Enable HOTPLUG support by default --- cloudinit/sources/DataSourceHetzner.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 13da635e516..8db8b020e6a 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -10,6 +10,7 @@ import cloudinit.sources.helpers.hetzner as hc_helper from cloudinit import dmi, net, sources, util +from cloudinit.event import EventScope, EventType from cloudinit.net.dhcp import NoDHCPLeaseError from cloudinit.net.ephemeral import EphemeralDHCPv4 @@ -40,6 +41,13 @@ class DataSourceHetzner(sources.DataSource): dsname = "Hetzner" + default_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.HOTPLUG, + } + } + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.distro = distro From d97988af906a8342279d0b840c84d5aa328d4724 Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Mon, 18 Aug 2025 13:55:11 +0000 Subject: [PATCH 03/12] Refactor Hetzner datasource to simplify network configuration handling --- cloudinit/sources/DataSourceHetzner.py | 101 +++++-------------------- cloudinit/sources/helpers/hetzner.py | 8 +- 2 files changed, 28 insertions(+), 81 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 8db8b020e6a..48df402cdab 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -38,7 +38,6 @@ class DataSourceHetzner(sources.DataSource): - dsname = "Hetzner" default_update_events = { @@ -164,87 +163,29 @@ def network_config(self): if not _net_config: raise RuntimeError("Unable to get meta-data from server....") - self._network_config_v2 = { - "version": 2, - "ethernets": {}, - } - self._network_config_v2["ethernets"] = self._network_config_v1_to_v2( - _net_config - ) - - for private_network in self.metadata.get("private-networks", []): - network_config_v2 = { - "match": { - "macaddress": private_network["mac_address"], - }, - "dhcp4": True, - } - idx = private_network["interface_num"] - # The key name (priv...) is just a virtual interface name. - # To rename the interface, "set-name" must be used, but we - # want to keep the OS-chosen name. - self._network_config_v2["ethernets"][ - f"priv{idx}" - ] = network_config_v2 - - return self._network_config_v2 - - def _network_config_v1_to_v2(self, network_config_v1): - ethernets = {} - - for network in network_config_v1["config"]: - networkv2 = { - "match": { - "macaddress": network["mac_address"], - }, - "set-name": network["name"], - "addresses": [], - "nameservers": { - "addresses": [], - }, - "routes": [], - } - - for subnet in network["subnets"]: - if ( - "ipv4" in subnet - and subnet["ipv4"] - and subnet["type"] == "dhcp" - ): - networkv2["dhcp4"] = True - if ( - "ipv6" in subnet - and subnet["ipv6"] - and subnet["type"] == "dhcp" - ): - networkv2["dhcp6"] = True - - if subnet["type"] == "static": - if "address" in subnet: - networkv2["addresses"].append(subnet["address"]) - - if "dns_nameservers" in subnet: - for ns in subnet["dns_nameservers"]: - networkv2["nameservers"]["addresses"].append(ns) - - target = None - if "ipv4" in subnet and subnet["ipv4"]: - target = "0.0.0.0/0" - elif "ipv6" in subnet and subnet["ipv6"]: - target = "::/0" - - if "gateway" in subnet and target: - networkv2["routes"].append( + _private_networks = self.metadata.get("private-networks", []) + _private_networks_config = [] + for _private_network in _private_networks: + _private_networks_config.extend( + [ + { + "type": "physical", + "mac_address": _private_network["mac_address"], + "name": hc_helper.get_interface_name_from_mac( + _private_network["mac_address"] + ), + "subnets": [ { - "on-link": True, - "to": target, - "via": subnet["gateway"], + "ipv4": True, + "type": "dhcp", } - ) - - ethernets[network["name"]] = networkv2 - - return ethernets + ], + } + ] + ) + _net_config["config"].extend(_private_networks_config) + self._network_config = _net_config + return self._network_config def get_hcloud_data(): diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index 50fbcb0468a..da073fe0cff 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -3,7 +3,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import url_helper, util +from cloudinit import url_helper, util, net def read_metadata(url, timeout=2, sec_between=2, retries=30): @@ -22,3 +22,9 @@ def read_userdata(url, timeout=2, sec_between=2, retries=30): if not response.ok(): raise RuntimeError("unable to read userdata at %s" % url) return response.contents + + +def get_interface_name_from_mac(mac: str) -> str | None: + mac_to_iface = net.get_interfaces_by_mac() + return mac_to_iface.get(mac.lower()) + From a4ef1b5c522bb16c59058f7d49839558e0fc56eb Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Tue, 19 Aug 2025 10:35:29 +0000 Subject: [PATCH 04/12] Add BOOT event support and refactor optional typing in Hetzner datasource --- cloudinit/sources/DataSourceHetzner.py | 1 + cloudinit/sources/helpers/hetzner.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 48df402cdab..040392ba409 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -43,6 +43,7 @@ class DataSourceHetzner(sources.DataSource): default_update_events = { EventScope.NETWORK: { EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, EventType.HOTPLUG, } } diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index da073fe0cff..ff9735825c8 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -4,6 +4,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import url_helper, util, net +from typing import Optional def read_metadata(url, timeout=2, sec_between=2, retries=30): @@ -24,7 +25,7 @@ def read_userdata(url, timeout=2, sec_between=2, retries=30): return response.contents -def get_interface_name_from_mac(mac: str) -> str | None: +def get_interface_name_from_mac(mac: str) -> Optional[str] : mac_to_iface = net.get_interfaces_by_mac() return mac_to_iface.get(mac.lower()) From f9dfe48fd41d404f9b10d9c174d96530036802e5 Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Sat, 23 Aug 2025 14:43:58 +0000 Subject: [PATCH 05/12] Refactor Hetzner datasource to support IPv6, streamline metadata retrieval, and improve timeout handling --- cloudinit/sources/DataSourceHetzner.py | 90 +++++++++++++++++--------- cloudinit/sources/helpers/hetzner.py | 52 +++++++++------ 2 files changed, 95 insertions(+), 47 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 040392ba409..c5257922967 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -9,25 +9,30 @@ import logging import cloudinit.sources.helpers.hetzner as hc_helper -from cloudinit import dmi, net, sources, util +from cloudinit import dmi, net, sources, util, url_helper from cloudinit.event import EventScope, EventType from cloudinit.net.dhcp import NoDHCPLeaseError -from cloudinit.net.ephemeral import EphemeralDHCPv4 +from cloudinit.net.ephemeral import EphemeralIPNetwork LOG = logging.getLogger(__name__) -BASE_URL_V1 = "http://169.254.169.254/hetzner/v1" +BASE_URLS_V1 = [ + f"http://[fe80::a9fe:a9fe%25{net.find_fallback_nic()}]/hetzner/v1/", + "http://169.254.169.254/hetzner/v1/", +] + BUILTIN_DS_CONFIG = { - "metadata_url": BASE_URL_V1 + "/metadata", - "metadata_private_networks_url": BASE_URL_V1 - + "/metadata/private-networks", - "userdata_url": BASE_URL_V1 + "/userdata", + "metadata_path": "metadata", + "metadata_private_networks_path": "metadata/private-networks", + "userdata_path": "userdata", } MD_RETRIES = 60 MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 +MD_MAX_WAIT = 120 +MD_SLEEP_TIME = 2 # Do not re-configure the network on non-Hetzner network interface # changes. Currently, Hetzner private network addresses start with 0x86. @@ -51,21 +56,23 @@ class DataSourceHetzner(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.distro = distro - self.metadata = dict() + self.metadata = {} self.ds_cfg = util.mergemanydict( [ util.get_cfg_by_path(sys_cfg, ["datasource", "Hetzner"], {}), BUILTIN_DS_CONFIG, ] ) - self.metadata_address = self.ds_cfg["metadata_url"] - self.metadata_private_networks_address = self.ds_cfg[ - "metadata_private_networks_url" + self.metadata_path = self.ds_cfg["metadata_path"] + self.metadata_private_networks_path = self.ds_cfg[ + "metadata_private_networks_path" ] - self.userdata_address = self.ds_cfg["userdata_url"] + self.userdata_path = self.ds_cfg["userdata_path"] self.retries = self.ds_cfg.get("retries", MD_RETRIES) self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT) self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY) + self.max_wait = self.ds_cfg.get("max_wait", MD_MAX_WAIT) + self.sleep_time = self.ds_cfg.get("sleep_time", MD_SLEEP_TIME) self._network_config = sources.UNSET self.dsmode = sources.DSMODE_NETWORK self.metadata_full = None @@ -79,33 +86,58 @@ def _get_data(self): return False try: - with EphemeralDHCPv4( + with EphemeralIPNetwork( self.distro, - iface=net.find_fallback_nic(), + interface=net.find_fallback_nic(), + ipv4=True, + ipv6=True, connectivity_urls_data=[ { - "url": BASE_URL_V1 + "/metadata/instance-id", + "url": url_helper.combine_url( + url, "metadata/instance-id" + ) } + for url in BASE_URLS_V1 ], ): - md = hc_helper.read_metadata( - self.metadata_address, + url, contents = hc_helper.get_metadata( + [ + url_helper.combine_url(url, self.metadata_path) + for url in BASE_URLS_V1 + ], + max_wait=self.max_wait, timeout=self.timeout, - sec_between=self.wait_retry, - retries=self.retries, + sleep_time=self.sleep_time, ) - md["private-networks"] = hc_helper.read_metadata( - self.metadata_private_networks_address, + LOG.debug("Using metadata source: '%s'", url) + md = util.load_yaml(contents.decode(), allowed=(dict, list)) + url, contents = hc_helper.get_metadata( + [ + url_helper.combine_url( + url, self.metadata_private_networks_path + ) + for url in BASE_URLS_V1 + ], + max_wait=self.max_wait, timeout=self.timeout, - sec_between=self.wait_retry, - retries=self.retries, + sleep_time=self.sleep_time, + ) + LOG.debug("Using private_networks source: '%s'", url) + md["private-networks"] = util.load_yaml( + contents.decode(), allowed=(dict, list) ) - ud = hc_helper.read_userdata( - self.userdata_address, + url, ud = hc_helper.get_metadata( + [ + url_helper.combine_url(url, self.userdata_path) + for url in BASE_URLS_V1 + ], + max_wait=self.max_wait, timeout=self.timeout, - sec_between=self.wait_retry, - retries=self.retries, + sleep_time=self.sleep_time, ) + LOG.debug("Using userdata source: '%s'", url) + if not ud: + LOG.debug("Got empty userdata") except NoDHCPLeaseError as e: LOG.error("Bailing, DHCP Exception: %s", e) raise @@ -192,7 +224,7 @@ def network_config(self): def get_hcloud_data(): vendor_name = dmi.read_dmi_data("system-manufacturer") if vendor_name != "Hetzner": - return (False, None) + return False, None serial = dmi.read_dmi_data("system-serial-number") if serial: @@ -200,7 +232,7 @@ def get_hcloud_data(): else: raise RuntimeError("Hetzner Cloud detected, but no serial found") - return (True, serial) + return True, serial # Used to match classes to dependencies diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index ff9735825c8..26f8a98f49f 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -3,29 +3,45 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import url_helper, util, net +from cloudinit import url_helper, net from typing import Optional -def read_metadata(url, timeout=2, sec_between=2, retries=30): - response = url_helper.readurl( - url, timeout=timeout, sec_between=sec_between, retries=retries +def skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool: + """Returns False if cause.code is 200 and 'Content-Length' is '0'.""" + return not ( + cause.code == 200 and cause.headers.get("Content-Length") == "0" ) - if not response.ok(): - raise RuntimeError("unable to read metadata at %s" % url) - return util.load_yaml(response.contents.decode(), allowed=(dict, list)) -def read_userdata(url, timeout=2, sec_between=2, retries=30): - response = url_helper.readurl( - url, timeout=timeout, sec_between=sec_between, retries=retries - ) - if not response.ok(): - raise RuntimeError("unable to read userdata at %s" % url) - return response.contents - - -def get_interface_name_from_mac(mac: str) -> Optional[str] : +def get_metadata( + urls, + max_wait=120, + sec_between=2, + retries=30, + timeout=2, + sleep_time=2, + exception_cb=None, +) -> tuple[str, bytes]: + try: + if not exception_cb: + # It is ok for userdata to not exist (thats why we are stopping if + # response is empty) and just in that case returning an empty + # string. + exception_cb = skip_retry_on_empty_response + url, contents = url_helper.wait_for_url( + urls=urls, + max_wait=max_wait, + timeout=timeout, + sleep_time=sleep_time, + exception_cb=exception_cb, + ) + return url, contents + except url_helper.UrlError as e: + if e.code == 200 and e.headers.get("Content-Length") == "0": + return e.url, b"" + + +def get_interface_name_from_mac(mac: str) -> Optional[str]: mac_to_iface = net.get_interfaces_by_mac() return mac_to_iface.get(mac.lower()) - From 54a0ccf223fb4e238176f3ba268b1296abe99ebb Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Mon, 25 Aug 2025 19:32:56 +0000 Subject: [PATCH 06/12] Refactor Hetzner unit tests to support expanded metadata, private networks, and improve mocking consistency --- tests/unittests/sources/test_hetzner.py | 63 ++++++++++++++++++------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index 5867a4fac91..84014d5069c 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -8,8 +8,7 @@ from cloudinit.sources import DataSourceHetzner from tests.unittests.helpers import CiTestCase, mock -METADATA = util.load_yaml( - """ +METADATA = b""" hostname: cloudinit-test instance-id: 123456 local-ipv4: '' @@ -43,13 +42,24 @@ test-key@workstation vendor_data: "test" """ -) USERDATA = b"""#cloud-config runcmd: - [touch, /root/cloud-init-worked ] """ +PRIVATE_NETWORKS = b""" +- ip: 10.1.0.2 + alias_ips: [] + interface_num: 2 + mac_address: 86:00:00:aa:5d:f8 + network_id: 11352901 + network_name: network-2 + network: 10.1.0.0/16 + subnet: 10.1.0.0/24 + gateway: 10.1.0.1 +""" + class TestDataSourceHetzner(CiTestCase): """ @@ -69,15 +79,13 @@ def get_ds(self): return ds @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") - @mock.patch("cloudinit.sources.DataSourceHetzner.EphemeralDHCPv4") + @mock.patch("cloudinit.sources.DataSourceHetzner.EphemeralIPNetwork") @mock.patch("cloudinit.net.find_fallback_nic") - @mock.patch("cloudinit.sources.helpers.hetzner.read_metadata") - @mock.patch("cloudinit.sources.helpers.hetzner.read_userdata") + @mock.patch("cloudinit.sources.helpers.hetzner.get_metadata") @mock.patch("cloudinit.sources.DataSourceHetzner.get_hcloud_data") def test_read_data( self, m_get_hcloud_data, - m_usermd, m_readmd, m_fallback_nic, m_net, @@ -85,10 +93,15 @@ def test_read_data( ): m_get_hcloud_data.return_value = ( True, - str(METADATA.get("instance-id")), + str(util.load_yaml(METADATA).get("instance-id")), ) - m_readmd.return_value = METADATA.copy() - m_usermd.return_value = USERDATA + # Use side_effect to return values for the three sequential calls to + # helpers.hetzner.get_metadata: metadata, private-networks, userdata + m_readmd.side_effect = [ + ("metadata_url", METADATA), + ("privnets_url", PRIVATE_NETWORKS), + ("userdata_url", USERDATA), + ] m_fallback_nic.return_value = "eth0" m_dhcp.return_value = [ { @@ -106,25 +119,43 @@ def test_read_data( m_net.assert_called_once_with( ds.distro, - iface="eth0", + interface="eth0", + ipv4=True, + ipv6=True, connectivity_urls_data=[ + { + "url": "http://[fe80::a9fe:a9fe%25eth0]/hetzner/v1/metadata/instance-id" + }, { "url": "http://169.254.169.254/hetzner/v1/metadata/instance-id" - } + }, ], ) self.assertTrue(m_readmd.called) - self.assertEqual(METADATA.get("hostname"), ds.get_hostname().hostname) + self.assertEqual( + util.load_yaml(METADATA).get("hostname"), + ds.get_hostname().hostname, + ) - self.assertEqual(METADATA.get("public-keys"), ds.get_public_ssh_keys()) + self.assertEqual( + util.load_yaml(METADATA).get("public-keys"), + ds.get_public_ssh_keys(), + ) + self.assertEqual( + ds.metadata["private-networks"], + util.load_yaml(PRIVATE_NETWORKS, allowed=(dict, list)), + ) self.assertIsInstance(ds.get_public_ssh_keys(), list) self.assertEqual(ds.get_userdata_raw(), USERDATA) - self.assertEqual(ds.get_vendordata_raw(), METADATA.get("vendor_data")) + self.assertEqual( + ds.get_vendordata_raw(), + util.load_yaml(METADATA).get("vendor_data"), + ) - @mock.patch("cloudinit.sources.helpers.hetzner.read_metadata") + @mock.patch("cloudinit.sources.helpers.hetzner.get_metadata") @mock.patch("cloudinit.net.find_fallback_nic") @mock.patch("cloudinit.sources.DataSourceHetzner.get_hcloud_data") def test_not_on_hetzner_returns_false( From fad5e5ad9f8b224488b8a9549b257c8c1bebed00 Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Mon, 1 Sep 2025 16:17:53 +0000 Subject: [PATCH 07/12] Switch to new API return code 204 on empty string --- cloudinit/sources/helpers/hetzner.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index 26f8a98f49f..0e22d890342 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -8,10 +8,8 @@ def skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool: - """Returns False if cause.code is 200 and 'Content-Length' is '0'.""" - return not ( - cause.code == 200 and cause.headers.get("Content-Length") == "0" - ) + """Returns False if cause.code is 204.""" + return cause.code != 204 def get_metadata( @@ -26,7 +24,7 @@ def get_metadata( try: if not exception_cb: # It is ok for userdata to not exist (thats why we are stopping if - # response is empty) and just in that case returning an empty + # HTTP code is 204) and just in that case returning an empty # string. exception_cb = skip_retry_on_empty_response url, contents = url_helper.wait_for_url( @@ -38,7 +36,7 @@ def get_metadata( ) return url, contents except url_helper.UrlError as e: - if e.code == 200 and e.headers.get("Content-Length") == "0": + if e.code == 204: return e.url, b"" From d6762f8012517bf2053676647b71b0fd25065cdf Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Tue, 2 Sep 2025 15:52:39 +0000 Subject: [PATCH 08/12] Refactor imports and replace unittest assertions with `assert` in Hetzner datasource and tests for consistency and readability # Conflicts: # tests/unittests/sources/test_hetzner.py --- cloudinit/sources/DataSourceHetzner.py | 2 +- cloudinit/sources/helpers/hetzner.py | 3 +- tests/unittests/sources/test_hetzner.py | 61 ++++++++++++------------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index c5257922967..09a0eb938d0 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -9,7 +9,7 @@ import logging import cloudinit.sources.helpers.hetzner as hc_helper -from cloudinit import dmi, net, sources, util, url_helper +from cloudinit import dmi, net, sources, url_helper, util from cloudinit.event import EventScope, EventType from cloudinit.net.dhcp import NoDHCPLeaseError from cloudinit.net.ephemeral import EphemeralIPNetwork diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index 0e22d890342..9ccea9492d1 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -3,9 +3,10 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import url_helper, net from typing import Optional +from cloudinit import net, url_helper + def skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool: """Returns False if cause.code is 204.""" diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index 84014d5069c..81646feb606 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -4,9 +4,11 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import helpers, settings, util +import pytest + +from cloudinit import settings, util from cloudinit.sources import DataSourceHetzner -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import mock METADATA = b""" hostname: cloudinit-test @@ -61,20 +63,17 @@ """ -class TestDataSourceHetzner(CiTestCase): +class TestDataSourceHetzner: """ Test reading the meta-data """ - def setUp(self): - super(TestDataSourceHetzner, self).setUp() - self.tmp = self.tmp_dir() - - def get_ds(self): + @pytest.fixture + def ds(self, paths, tmp_path): distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir + distro.get_tmp_exec_path = str(tmp_path) ds = DataSourceHetzner.DataSourceHetzner( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, distro, paths ) return ds @@ -90,6 +89,7 @@ def test_read_data( m_fallback_nic, m_net, m_dhcp, + ds, ): m_get_hcloud_data.return_value = ( True, @@ -113,9 +113,7 @@ def test_read_data( } ] - ds = self.get_ds() - ret = ds.get_data() - self.assertTrue(ret) + assert True is ds.get_data() m_net.assert_called_once_with( ds.distro, @@ -132,42 +130,39 @@ def test_read_data( ], ) - self.assertTrue(m_readmd.called) + assert 0 != m_readmd.call_count - self.assertEqual( - util.load_yaml(METADATA).get("hostname"), - ds.get_hostname().hostname, + assert ( + util.load_yaml(METADATA).get("hostname") + == ds.get_hostname().hostname ) - self.assertEqual( - util.load_yaml(METADATA).get("public-keys"), - ds.get_public_ssh_keys(), + assert ( + util.load_yaml(METADATA).get("public-keys") + == ds.get_public_ssh_keys() ) - self.assertEqual( - ds.metadata["private-networks"], - util.load_yaml(PRIVATE_NETWORKS, allowed=(dict, list)), + assert ds.metadata["private-networks"] == util.load_yaml( + PRIVATE_NETWORKS, allowed=(dict, list) ) - self.assertIsInstance(ds.get_public_ssh_keys(), list) - self.assertEqual(ds.get_userdata_raw(), USERDATA) - self.assertEqual( - ds.get_vendordata_raw(), - util.load_yaml(METADATA).get("vendor_data"), + assert isinstance(ds.get_public_ssh_keys(), list) + assert ds.get_userdata_raw() == USERDATA + assert ds.get_vendordata_raw() == util.load_yaml(METADATA).get( + "vendor_data" ) @mock.patch("cloudinit.sources.helpers.hetzner.get_metadata") @mock.patch("cloudinit.net.find_fallback_nic") @mock.patch("cloudinit.sources.DataSourceHetzner.get_hcloud_data") def test_not_on_hetzner_returns_false( - self, m_get_hcloud_data, m_find_fallback, m_read_md + self, m_get_hcloud_data, m_find_fallback, m_read_md, ds ): """If helper 'get_hcloud_data' returns False, return False from get_data.""" m_get_hcloud_data.return_value = (False, None) - ds = self.get_ds() ret = ds.get_data() - self.assertFalse(ret) + assert not ret # These are a white box attempt to ensure it did not search. - m_find_fallback.assert_not_called() - m_read_md.assert_not_called() + assert 0 == m_find_fallback.call_count + assert 0 == m_read_md.call_count From 8c7fed5628fe2445d67f745847b48eb4f171fded Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Tue, 2 Sep 2025 15:35:37 +0000 Subject: [PATCH 09/12] fix(ci): apply suggestions --- cloudinit/sources/DataSourceHetzner.py | 34 ++++++++++++++++++-------- cloudinit/sources/helpers/hetzner.py | 9 ++++--- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 09a0eb938d0..6d8d77d63ba 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -16,12 +16,6 @@ LOG = logging.getLogger(__name__) -BASE_URLS_V1 = [ - f"http://[fe80::a9fe:a9fe%25{net.find_fallback_nic()}]/hetzner/v1/", - "http://169.254.169.254/hetzner/v1/", -] - - BUILTIN_DS_CONFIG = { "metadata_path": "metadata", "metadata_private_networks_path": "metadata/private-networks", @@ -42,6 +36,13 @@ """ +def base_urls_v1(): + return ( + f"http://[fe80::a9fe:a9fe%25{net.find_fallback_nic()}]/hetzner/v1/", + "http://169.254.169.254/hetzner/v1/", + ) + + class DataSourceHetzner(sources.DataSource): dsname = "Hetzner" @@ -79,12 +80,25 @@ def __init__(self, sys_cfg, distro, paths): self.extra_hotplug_udev_rules = EXTRA_HOTPLUG_UDEV_RULES + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + self.extra_hotplug_udev_rules = EXTRA_HOTPLUG_UDEV_RULES + self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY) + self.max_wait = self.ds_cfg.get("max_wait", MD_MAX_WAIT) + self.sleep_time = self.ds_cfg.get("sleep_time", MD_SLEEP_TIME) + self.metadata_path = self.ds_cfg["metadata_path"] + self.metadata_private_networks_path = self.ds_cfg[ + "metadata_private_networks_path" + ] + self.userdata_path = self.ds_cfg["userdata_path"] + def _get_data(self): (on_hetzner, serial) = get_hcloud_data() if not on_hetzner: return False + base_urls = base_urls_v1() try: with EphemeralIPNetwork( self.distro, @@ -97,13 +111,13 @@ def _get_data(self): url, "metadata/instance-id" ) } - for url in BASE_URLS_V1 + for url in base_urls ], ): url, contents = hc_helper.get_metadata( [ url_helper.combine_url(url, self.metadata_path) - for url in BASE_URLS_V1 + for url in base_urls ], max_wait=self.max_wait, timeout=self.timeout, @@ -116,7 +130,7 @@ def _get_data(self): url_helper.combine_url( url, self.metadata_private_networks_path ) - for url in BASE_URLS_V1 + for url in base_urls ], max_wait=self.max_wait, timeout=self.timeout, @@ -129,7 +143,7 @@ def _get_data(self): url, ud = hc_helper.get_metadata( [ url_helper.combine_url(url, self.userdata_path) - for url in BASE_URLS_V1 + for url in base_urls ], max_wait=self.max_wait, timeout=self.timeout, diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index 9ccea9492d1..ba5bd975246 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -3,7 +3,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from typing import Optional +from typing import Optional, Tuple from cloudinit import net, url_helper @@ -16,12 +16,10 @@ def skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool: def get_metadata( urls, max_wait=120, - sec_between=2, - retries=30, timeout=2, sleep_time=2, exception_cb=None, -) -> tuple[str, bytes]: +) -> Tuple[Optional[str], bytes]: try: if not exception_cb: # It is ok for userdata to not exist (thats why we are stopping if @@ -35,10 +33,13 @@ def get_metadata( sleep_time=sleep_time, exception_cb=exception_cb, ) + if not url: + raise RuntimeError("No data received from urls: '%s:", urls) return url, contents except url_helper.UrlError as e: if e.code == 204: return e.url, b"" + raise def get_interface_name_from_mac(mac: str) -> Optional[str]: From 135ae07137df985d7c30352e31d051713da4fc5f Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Tue, 2 Sep 2025 19:00:40 +0000 Subject: [PATCH 10/12] fix(ci): apply suggestions --- cloudinit/sources/helpers/hetzner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index ba5bd975246..99a3712e318 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -34,7 +34,7 @@ def get_metadata( exception_cb=exception_cb, ) if not url: - raise RuntimeError("No data received from urls: '%s:", urls) + raise RuntimeError("No data received from urls: '%s':" % urls) return url, contents except url_helper.UrlError as e: if e.code == 204: From 3b20582108799321a7f8ece3689a6ec278276cab Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Tue, 2 Sep 2025 19:03:50 +0000 Subject: [PATCH 11/12] feat(doc): Add Hetzner to list of supported providers for hotplug --- doc/module-docs/cc_install_hotplug/data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/module-docs/cc_install_hotplug/data.yaml b/doc/module-docs/cc_install_hotplug/data.yaml index 2277c9f274a..abdac913a46 100644 --- a/doc/module-docs/cc_install_hotplug/data.yaml +++ b/doc/module-docs/cc_install_hotplug/data.yaml @@ -14,7 +14,7 @@ cc_install_hotplug: around this limitation, one can wait until cloud-init has completed before hotplugging devices. - Currently supported datasources: Openstack, EC2 + Currently supported datasources: Openstack, EC2, Hetzner examples: - comment: | Example 1: Enable hotplug of network devices From f57e6c26d98727fd53f6db627e6ceafe20cb5692 Mon Sep 17 00:00:00 2001 From: Philipp Bender Date: Thu, 4 Sep 2025 22:34:28 +0000 Subject: [PATCH 12/12] apply suggestions --- cloudinit/sources/DataSourceHetzner.py | 46 ++++++++++++------------- cloudinit/sources/helpers/hetzner.py | 14 +++----- tests/unittests/sources/test_hetzner.py | 10 +++--- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 6d8d77d63ba..2aaba2a5435 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -108,7 +108,7 @@ def _get_data(self): connectivity_urls_data=[ { "url": url_helper.combine_url( - url, "metadata/instance-id" + url, f"{self.metadata_path}/instance-id" ) } for url in base_urls @@ -206,32 +206,30 @@ def network_config(self): if self._network_config != sources.UNSET: return self._network_config - _net_config = self.metadata["network-config"] - if not _net_config: + net_config = self.metadata["network-config"] + if not net_config: raise RuntimeError("Unable to get meta-data from server....") - _private_networks = self.metadata.get("private-networks", []) - _private_networks_config = [] - for _private_network in _private_networks: - _private_networks_config.extend( - [ - { - "type": "physical", - "mac_address": _private_network["mac_address"], - "name": hc_helper.get_interface_name_from_mac( - _private_network["mac_address"] - ), - "subnets": [ - { - "ipv4": True, - "type": "dhcp", - } - ], - } - ] + private_networks = self.metadata.get("private-networks", []) + private_networks_config = [] + for private_network in private_networks: + private_networks_config.append( + { + "type": "physical", + "mac_address": private_network["mac_address"], + "name": hc_helper.get_interface_name_from_mac( + private_network["mac_address"] + ), + "subnets": [ + { + "ipv4": True, + "type": "dhcp", + } + ], + } ) - _net_config["config"].extend(_private_networks_config) - self._network_config = _net_config + net_config["config"].extend(private_networks_config) + self._network_config = net_config return self._network_config diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index 99a3712e318..68abd4dc48d 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -8,8 +8,7 @@ from cloudinit import net, url_helper -def skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool: - """Returns False if cause.code is 204.""" +def _skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool: return cause.code != 204 @@ -18,20 +17,17 @@ def get_metadata( max_wait=120, timeout=2, sleep_time=2, - exception_cb=None, ) -> Tuple[Optional[str], bytes]: try: - if not exception_cb: - # It is ok for userdata to not exist (thats why we are stopping if - # HTTP code is 204) and just in that case returning an empty - # string. - exception_cb = skip_retry_on_empty_response url, contents = url_helper.wait_for_url( urls=urls, max_wait=max_wait, timeout=timeout, sleep_time=sleep_time, - exception_cb=exception_cb, + # It is ok for userdata to not exist (that's why we are stopping if + # HTTP code is 204) and just in that case returning an empty + # string. + exception_cb=_skip_retry_on_empty_response, ) if not url: raise RuntimeError("No data received from urls: '%s':" % urls) diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index 81646feb606..af0a893cd0f 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -85,7 +85,7 @@ def ds(self, paths, tmp_path): def test_read_data( self, m_get_hcloud_data, - m_readmd, + m_get_metadata, m_fallback_nic, m_net, m_dhcp, @@ -97,7 +97,7 @@ def test_read_data( ) # Use side_effect to return values for the three sequential calls to # helpers.hetzner.get_metadata: metadata, private-networks, userdata - m_readmd.side_effect = [ + m_get_metadata.side_effect = [ ("metadata_url", METADATA), ("privnets_url", PRIVATE_NETWORKS), ("userdata_url", USERDATA), @@ -130,7 +130,7 @@ def test_read_data( ], ) - assert 0 != m_readmd.call_count + assert 0 != m_get_metadata.call_count assert ( util.load_yaml(METADATA).get("hostname") @@ -155,7 +155,7 @@ def test_read_data( @mock.patch("cloudinit.net.find_fallback_nic") @mock.patch("cloudinit.sources.DataSourceHetzner.get_hcloud_data") def test_not_on_hetzner_returns_false( - self, m_get_hcloud_data, m_find_fallback, m_read_md, ds + self, m_get_hcloud_data, m_find_fallback, m_get_metadata, ds ): """If helper 'get_hcloud_data' returns False, return False from get_data.""" @@ -165,4 +165,4 @@ def test_not_on_hetzner_returns_false( assert not ret # These are a white box attempt to ensure it did not search. assert 0 == m_find_fallback.call_count - assert 0 == m_read_md.call_count + assert 0 == m_get_metadata.call_count