feat(hetzner): enable hotplug support and prepare IPv6 integration

cloudinit/sources/DataSourceHetzner.py

@@ -9,81 +9,149 @@
 import logging
 
 import cloudinit.sources.helpers.hetzner as hc_helper
-from cloudinit import dmi, net, sources, util
+from cloudinit import dmi, net, sources, url_helper, util
+from cloudinit.event import EventScope, EventType
 from cloudinit.net.dhcp import NoDHCPLeaseError
-from cloudinit.net.ephemeral import EphemeralDHCPv4
+from cloudinit.net.ephemeral import EphemeralIPNetwork
 
 LOG = logging.getLogger(__name__)
 
-BASE_URL_V1 = "http://169.254.169.254/hetzner/v1"
-
 BUILTIN_DS_CONFIG = {
-    "metadata_url": BASE_URL_V1 + "/metadata",
-    "userdata_url": BASE_URL_V1 + "/userdata",
+    "metadata_path": "metadata",
+    "metadata_private_networks_path": "metadata/private-networks",
+    "userdata_path": "userdata",
 }
 
 MD_RETRIES = 60
 MD_TIMEOUT = 2
 MD_WAIT_RETRY = 2
+MD_MAX_WAIT = 120
+MD_SLEEP_TIME = 2
 
+# Do not re-configure the network on non-Hetzner network interface
+# changes. Currently, Hetzner private network addresses start with 0x86.
+EXTRA_HOTPLUG_UDEV_RULES = """
+SUBSYSTEM=="net", ATTR{address}=="86:*", GOTO="cloudinit_hook"
+GOTO="cloudinit_end"
+"""
 
-class DataSourceHetzner(sources.DataSource):
 
+def base_urls_v1():
+    return (
+        f"http://[fe80::a9fe:a9fe%25{net.find_fallback_nic()}]/hetzner/v1/",
+        "http://169.254.169.254/hetzner/v1/",
+    )
+
+
+class DataSourceHetzner(sources.DataSource):
     dsname = "Hetzner"
 
+    default_update_events = {
+        EventScope.NETWORK: {
+            EventType.BOOT_NEW_INSTANCE,
+            EventType.BOOT,
+            EventType.HOTPLUG,
+        }
+    }
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.distro = distro
-        self.metadata = dict()
+        self.metadata = {}
         self.ds_cfg = util.mergemanydict(
             [
                 util.get_cfg_by_path(sys_cfg, ["datasource", "Hetzner"], {}),
                 BUILTIN_DS_CONFIG,
             ]
         )
-        self.metadata_address = self.ds_cfg["metadata_url"]
-        self.userdata_address = self.ds_cfg["userdata_url"]
+        self.metadata_path = self.ds_cfg["metadata_path"]
+        self.metadata_private_networks_path = self.ds_cfg[
+            "metadata_private_networks_path"
+        ]
+        self.userdata_path = self.ds_cfg["userdata_path"]
         self.retries = self.ds_cfg.get("retries", MD_RETRIES)
         self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT)
         self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY)
+        self.max_wait = self.ds_cfg.get("max_wait", MD_MAX_WAIT)
+        self.sleep_time = self.ds_cfg.get("sleep_time", MD_SLEEP_TIME)
         self._network_config = sources.UNSET
         self.dsmode = sources.DSMODE_NETWORK
         self.metadata_full = None
 
+        self.extra_hotplug_udev_rules = EXTRA_HOTPLUG_UDEV_RULES
+
+    def _unpickle(self, ci_pkl_version: int) -> None:
+        super()._unpickle(ci_pkl_version)
+        self.extra_hotplug_udev_rules = EXTRA_HOTPLUG_UDEV_RULES
+        self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY)
+        self.max_wait = self.ds_cfg.get("max_wait", MD_MAX_WAIT)
+        self.sleep_time = self.ds_cfg.get("sleep_time", MD_SLEEP_TIME)
+        self.metadata_path = self.ds_cfg["metadata_path"]
+        self.metadata_private_networks_path = self.ds_cfg[
+            "metadata_private_networks_path"
+        ]
+        self.userdata_path = self.ds_cfg["userdata_path"]
+
     def _get_data(self):
         (on_hetzner, serial) = get_hcloud_data()
 
         if not on_hetzner:
             return False
 
+        base_urls = base_urls_v1()
         try:
-            with EphemeralDHCPv4(
+            with EphemeralIPNetwork(
                 self.distro,
-                iface=net.find_fallback_nic(),
+                interface=net.find_fallback_nic(),
+                ipv4=True,
+                ipv6=True,
                 connectivity_urls_data=[
                     {
-                        "url": BASE_URL_V1 + "/metadata/instance-id",
+                        "url": url_helper.combine_url(
+                            url, f"{self.metadata_path}/instance-id"
+                        )
                     }
+                    for url in base_urls
                 ],
             ):
-                md = hc_helper.read_metadata(
-                    self.metadata_address,
+                url, contents = hc_helper.get_metadata(
+                    [
+                        url_helper.combine_url(url, self.metadata_path)
+                        for url in base_urls
+                    ],
+                    max_wait=self.max_wait,
                     timeout=self.timeout,
-                    sec_between=self.wait_retry,
-                    retries=self.retries,
+                    sleep_time=self.sleep_time,
                 )
-                ud = hc_helper.read_userdata(
-                    self.userdata_address,
+                LOG.debug("Using metadata source: '%s'", url)
+                md = util.load_yaml(contents.decode(), allowed=(dict, list))
+                url, contents = hc_helper.get_metadata(
+                    [
+                        url_helper.combine_url(
+                            url, self.metadata_private_networks_path
+                        )
+                        for url in base_urls
+                    ],
+                    max_wait=self.max_wait,
                     timeout=self.timeout,
-                    sec_between=self.wait_retry,
-                    retries=self.retries,
+                    sleep_time=self.sleep_time,
                 )
-                pn = hc_helper.read_metadata(
-                    self.metadata_address + "/private-networks",
+                LOG.debug("Using private_networks source: '%s'", url)
+                md["private-networks"] = util.load_yaml(
+                    contents.decode(), allowed=(dict, list)
+                )
+                url, ud = hc_helper.get_metadata(
+                    [
+                        url_helper.combine_url(url, self.userdata_path)
+                        for url in base_urls
+                    ],
+                    max_wait=self.max_wait,
                     timeout=self.timeout,
-                    sec_between=self.wait_retry,
-                    retries=self.retries,
+                    sleep_time=self.sleep_time,
                 )
+                LOG.debug("Using userdata source: '%s'", url)
+                if not ud:
+                    LOG.debug("Got empty userdata")
         except NoDHCPLeaseError as e:
             LOG.error("Bailing, DHCP Exception: %s", e)
             raise
@@ -105,7 +173,7 @@ def _get_data(self):
         self.metadata["local-hostname"] = md["hostname"]
         self.metadata["network-config"] = md.get("network-config", None)
         self.metadata["public-keys"] = md.get("public-keys", None)
-        self.metadata["private-networks"] = pn
+        self.metadata["private-networks"] = md.get("private-networks", [])
         self.vendordata_raw = md.get("vendor_data", None)
 
         # instance-id and serial from SMBIOS should be identical
@@ -138,27 +206,45 @@ def network_config(self):
         if self._network_config != sources.UNSET:
             return self._network_config
 
-        _net_config = self.metadata["network-config"]
-        if not _net_config:
+        net_config = self.metadata["network-config"]
+        if not net_config:
             raise RuntimeError("Unable to get meta-data from server....")
 
-        self._network_config = _net_config
-
+        private_networks = self.metadata.get("private-networks", [])
+        private_networks_config = []
+        for private_network in private_networks:
+            private_networks_config.append(
+                {
+                    "type": "physical",
+                    "mac_address": private_network["mac_address"],
+                    "name": hc_helper.get_interface_name_from_mac(
+                        private_network["mac_address"]
+                    ),
+                    "subnets": [
+                        {
+                            "ipv4": True,
+                            "type": "dhcp",
+                        }
+                    ],
+                }
+            )
+        net_config["config"].extend(private_networks_config)
+        self._network_config = net_config
         return self._network_config
 
 
 def get_hcloud_data():
     vendor_name = dmi.read_dmi_data("system-manufacturer")
     if vendor_name != "Hetzner":
-        return (False, None)
+        return False, None
 
     serial = dmi.read_dmi_data("system-serial-number")
     if serial:
         LOG.debug("Running on Hetzner Cloud: serial=%s", serial)
     else:
         raise RuntimeError("Hetzner Cloud detected, but no serial found")
 
-    return (True, serial)
+    return True, serial
 
 
 # Used to match classes to dependencies

cloudinit/sources/helpers/hetzner.py

@@ -3,22 +3,41 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit import url_helper, util
+from typing import Optional, Tuple
 
+from cloudinit import net, url_helper
 
-def read_metadata(url, timeout=2, sec_between=2, retries=30):
-    response = url_helper.readurl(
-        url, timeout=timeout, sec_between=sec_between, retries=retries
-    )
-    if not response.ok():
-        raise RuntimeError("unable to read metadata at %s" % url)
-    return util.load_yaml(response.contents.decode(), allowed=(dict, list))
 
+def _skip_retry_on_empty_response(cause: url_helper.UrlError) -> bool:
+    return cause.code != 204
 
-def read_userdata(url, timeout=2, sec_between=2, retries=30):
-    response = url_helper.readurl(
-        url, timeout=timeout, sec_between=sec_between, retries=retries
-    )
-    if not response.ok():
-        raise RuntimeError("unable to read userdata at %s" % url)
-    return response.contents
+
+def get_metadata(
+    urls,
+    max_wait=120,
+    timeout=2,
+    sleep_time=2,
+) -> Tuple[Optional[str], bytes]:
+    try:
+        url, contents = url_helper.wait_for_url(
+            urls=urls,
+            max_wait=max_wait,
+            timeout=timeout,
+            sleep_time=sleep_time,
+            # It is ok for userdata to not exist (that's why we are stopping if
+            # HTTP code is 204) and just in that case returning an empty
+            # string.
+            exception_cb=_skip_retry_on_empty_response,
+        )
+        if not url:
+            raise RuntimeError("No data received from urls: '%s':" % urls)
+        return url, contents
+    except url_helper.UrlError as e:
+        if e.code == 204:
+            return e.url, b""
+        raise
+
+
+def get_interface_name_from_mac(mac: str) -> Optional[str]:
+    mac_to_iface = net.get_interfaces_by_mac()
+    return mac_to_iface.get(mac.lower())

doc/module-docs/cc_install_hotplug/data.yaml

@@ -14,7 +14,7 @@ cc_install_hotplug:
     around this limitation, one can wait until cloud-init has completed
     before hotplugging devices.
 
-    Currently supported datasources: Openstack, EC2
+    Currently supported datasources: Openstack, EC2, Hetzner
   examples:
   - comment: |
       Example 1: Enable hotplug of network devices