From 7b5ca7a9493333a8bc0f0abb1d5dbba3050b82b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Wed, 9 Dec 2020 20:47:28 +0200 Subject: [PATCH 01/16] New datasource: UpCloud --- cloudinit/apport.py | 1 + cloudinit/settings.py | 1 + cloudinit/sources/DataSourceUpCloud.py | 142 +++++++++++++++++ cloudinit/sources/helpers/upcloud.py | 207 +++++++++++++++++++++++++ 4 files changed, 351 insertions(+) create mode 100644 cloudinit/sources/DataSourceUpCloud.py create mode 100644 cloudinit/sources/helpers/upcloud.py diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 9bded16c697..25f254e36ee 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -39,6 +39,7 @@ 'SAP Converged Cloud', 'Scaleway', 'SmartOS', + 'UpCloud', 'VMware', 'ZStack', 'Other' diff --git a/cloudinit/settings.py b/cloudinit/settings.py index ca4ffa8e681..bd0cd9f73c0 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -41,6 +41,7 @@ 'Oracle', 'Exoscale', 'RbxCloud', + 'UpCloud', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py new file mode 100644 index 00000000000..538ce5b9df6 --- /dev/null +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -0,0 +1,142 @@ +# Author: Antti Myyrä +# +# This file is part of cloud-init. See LICENSE file for license information. + +# UpCloud server metadata API: +# https://developers.upcloud.com/1.3/8-servers/#metadata-service + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util +from cloudinit import net as cloudnet +from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError + + +from cloudinit.sources.helpers import upcloud as uc_helper + +LOG = logging.getLogger(__name__) + +BUILTIN_DS_CONFIG = {"metadata_url": "http://169.254.169.254/metadata/v1.json"} + +# Wait for a up to a minute, retrying the meta-data server +# every 2 seconds. +MD_RETRIES = 30 +MD_TIMEOUT = 2 +MD_WAIT_RETRY = 2 + + +class DataSourceUpCloud(sources.DataSource): + + dsname = "UpCloud" + + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.distro = distro + self.metadata = dict() + self.ds_cfg = util.mergemanydict( + [ + util.get_cfg_by_path(sys_cfg, ["datasource", "UpCloud"], {}), + BUILTIN_DS_CONFIG, + ] + ) + self.metadata_address = self.ds_cfg["metadata_url"] + self.retries = self.ds_cfg.get("retries", MD_RETRIES) + self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT) + self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY) + self.perform_dhcp_setup = self.ds_cfg.get("dhcp_setup", True) + self._network_config = None + + def _get_sysinfo(self): + return uc_helper.read_sysinfo() + + def _read_metadata(self): + return uc_helper.read_metadata( + self.metadata_address, + timeout=self.timeout, + sec_between=self.wait_retry, + retries=self.retries, + ) + + def _get_data(self): + (is_upcloud, server_uuid) = self._get_sysinfo() + + # only proceed if we know we are on UpCloud + if not is_upcloud: + return False + + LOG.info("Running on UpCloud. server_uuid=%s", server_uuid) + + if self.perform_dhcp_setup: # Setup networking in init-local stage. + try: + LOG.debug("Finding a fallback NIC") + nic = cloudnet.find_fallback_nic() + LOG.debug("Discovering metadata via DHCP interface %s", nic) + with EphemeralDHCPv4(nic): + md = util.log_time( + logfunc=LOG.debug, + msg="Reading from metadata service", + func=self._read_metadata, + ) + except (NoDHCPLeaseError, sources.InvalidMetaDataException) as e: + util.logexc(LOG, str(e)) + return False + else: + try: + LOG.debug("Discovering metadata without DHCP-configured networking") + md = util.log_time( + logfunc=LOG.debug, + msg="Reading from metadata service", + func=self._read_metadata, + ) + except sources.InvalidMetaDataException as e: + util.logexc(LOG, str(e)) + LOG.info( + "No DHCP-enabled interfaces available, unable to fetch metadata for %s", + server_uuid, + ) + return False + + self.metadata_full = md + self.metadata["instance-id"] = md.get("instance_id", server_uuid) + self.metadata["local-hostname"] = md.get("hostname") + self.metadata["network"] = md.get("network") + self.metadata["public-keys"] = md.get("public_keys") + self.metadata["availability_zone"] = md.get("region", "default") + self.vendordata_raw = md.get("vendor_data", None) + self.userdata_raw = md.get("user_data", None) + + return True + + def check_instance_id(self, sys_cfg): + return sources.instance_id_matches_system_uuid(self.get_instance_id()) + + @property + def network_config(self): + """Configure the networking. This needs to be done each boot, + since the IP and interface information might have changed + due to reconfiguration. + """ + + if self._network_config: + return self._network_config + + raw_network_config = self.metadata.get("network") + if not raw_network_config: + raise Exception("Unable to get network meta-data from server....") + + self._network_config = uc_helper.convert_to_network_config_v1( + raw_network_config + ) + return self._network_config + + +# Used to match classes to dependencies +datasources = [(DataSourceUpCloud, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK))] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py new file mode 100644 index 00000000000..5ce3cb63213 --- /dev/null +++ b/cloudinit/sources/helpers/upcloud.py @@ -0,0 +1,207 @@ +# Author: Antti Myyrä +# +# This file is part of cloud-init. See LICENSE file for license information. + +import json + +from cloudinit import dmi +from cloudinit import log as logging +from cloudinit import net as cloudnet +from cloudinit import url_helper + +LOG = logging.getLogger(__name__) + + +def convert_to_network_config_v1(config): + """Convert the UpCloud network metadata description into + Cloud-init's version 1 netconfig format. + + Example JSON: + "interfaces": [ + { + "index": 1, + "ip_addresses": [ + { + "address": "94.237.105.53", + "dhcp": true, + "dns": [ + "94.237.127.9", + "94.237.40.9" + ], + "family": "IPv4", + "floating": false, + "gateway": "94.237.104.1", + "network": "94.237.104.0/22" + }, + { + "address": "94.237.105.50", + "dhcp": false, + "dns": [], + "family": "IPv4", + "floating": true, + "gateway": "", + "network": "94.237.105.50/32" + } + ], + "mac": "32:d5:ba:4a:36:e7", + "network_id": "031457f4-0f8c-483c-96f2-eccede02909c", + "type": "public" + }, + { + "index": 2, + "ip_addresses": [ + { + "address": "10.6.3.27", + "dhcp": true, + "dns": [], + "family": "IPv4", + "floating": false, + "gateway": "10.6.0.1", + "network": "10.6.0.0/22" + } + ], + "mac": "32:d5:ba:4a:84:cc", + "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1", + "type": "utility" + }, + { + "index": 3, + "ip_addresses": [ + { + "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7", + "dhcp": true, + "dns": [ + "2a04:3540:53::1", + "2a04:3544:53::1" + ], + "family": "IPv6", + "floating": false, + "gateway": "2a04:3545:1000:720::1", + "network": "2a04:3545:1000:720::/64" + } + ], + "mac": "32:d5:ba:4a:63:e7", + "network_id": "03000000-0000-4000-8046-000000000000", + "type": "public" + }, + { + "index": 4, + "ip_addresses": [ + { + "address": "172.30.1.10", + "dhcp": true, + "dns": [], + "family": "IPv4", + "floating": false, + "gateway": "172.30.1.1", + "network": "172.30.1.0/24" + } + ], + "mac": "32:d5:ba:4a:8a:e1", + "network_id": "035a0a4a-77b4-4de5-820d-189fc8135714", + "type": "private" + } + ], + "dns": [ + "94.237.127.9", + "94.237.40.9" + ] + } + """ + + def _get_subnet_config(ip_addr, dns): + if ip_addr.get("dhcp"): + return {"type": "dhcp6" if ip_addr.get("family") == "IPv6" else "dhcp"} + + subpart = { + "type": "static6" if ip_addr.get("family") == "IPv6" else "static", + "control": "auto", + "address": ip_addr.get("address"), + } + + if ip_addr.get("gateway"): + subpart["gateway"] = ip_addr.get("gateway") + + if "/" in ip_addr.get("network"): + subpart["netmask"] = ip_addr.get("network").split("/")[1] + + if dns != ip_addr.get("dns") and ip_addr.get("dns"): + subpart["dns_nameservers"] = ip_addr.get("dns") + + return subpart + + nic_configs = [] + macs_to_interfaces = cloudnet.get_interfaces_by_mac() + LOG.debug("NIC mapping: %s", macs_to_interfaces) + + for raw_iface in config.get("interfaces"): + LOG.debug("Considering %s", raw_iface) + + mac_address = raw_iface.get("mac") + if mac_address not in macs_to_interfaces: + raise RuntimeError( + "Did not find network interface on system " + "with mac '%s'. Cannot apply configuration: %s" + % (mac_address, raw_iface) + ) + + iface_type = raw_iface.get("type") + sysfs_name = macs_to_interfaces.get(mac_address) + + LOG.debug( + "Found %s interface '%s' with address '%s' (index %d)", + iface_type, + sysfs_name, + mac_address, + raw_iface.get("index"), + ) + + interface = {"type": "physical", "name": sysfs_name, "mac_address": mac_address} + + subnets = [] + for ip_address in raw_iface.get("ip_addresses"): + sub_part = _get_subnet_config(ip_address, config.get("dns")) + subnets.append(sub_part) + + interface["subnets"] = subnets + nic_configs.append(interface) + + if config.get("dns"): + LOG.debug("Setting DNS nameservers to %s", config.get("dns")) + nic_configs.append({"type": "nameserver", "address": config.get("dns")}) + + return {"version": 1, "config": nic_configs} + + +def read_metadata(url, timeout=2, sec_between=2, retries=30): + response = url_helper.readurl( + url, timeout=timeout, sec_between=sec_between, retries=retries + ) + if not response.ok(): + raise RuntimeError("unable to read metadata at %s" % url) + return json.loads(response.contents.decode()) + + +def read_sysinfo(): + # UpCloud embeds vendor ID and server UUID in the + # SMBIOS information + + # Detect if we are on UpCloud and return the UUID + + vendor_name = dmi.read_dmi_data("system-manufacturer") + if vendor_name != "UpCloud": + return False, None + + server_uuid = dmi.read_dmi_data("system-uuid") + if server_uuid: + LOG.debug("system identified via SMBIOS as UpCloud server: %s", server_uuid) + else: + msg = ( + "system identified via SMBIOS as a UpCloud server, but " + "did not provide an ID. Please contact support via" + "https://hub.upcloud.com or via email with support@upcloud.com" + ) + LOG.critical(msg) + raise RuntimeError(msg) + + return True, server_uuid From 806b711cbea1440ac24facca09a0b7e9fc8ed352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 15 Dec 2020 20:02:54 +0200 Subject: [PATCH 02/16] DataSourceUpCloud: small styling fixes --- cloudinit/sources/DataSourceUpCloud.py | 10 +++++++--- cloudinit/sources/helpers/upcloud.py | 26 ++++++++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index 538ce5b9df6..3051d71f883 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -82,7 +82,9 @@ def _get_data(self): return False else: try: - LOG.debug("Discovering metadata without DHCP-configured networking") + LOG.debug( + "Discovering metadata without DHCP-configured networking" + ) md = util.log_time( logfunc=LOG.debug, msg="Reading from metadata service", @@ -91,7 +93,8 @@ def _get_data(self): except sources.InvalidMetaDataException as e: util.logexc(LOG, str(e)) LOG.info( - "No DHCP-enabled interfaces available, unable to fetch metadata for %s", + "No DHCP-enabled interfaces available, " + "unable to fetch metadata for %s", server_uuid, ) return False @@ -131,7 +134,8 @@ def network_config(self): # Used to match classes to dependencies -datasources = [(DataSourceUpCloud, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK))] +datasources = [(DataSourceUpCloud, + (sources.DEP_FILESYSTEM, sources.DEP_NETWORK))] # Return a list of data sources that match this set of dependencies diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py index 5ce3cb63213..5d287ebde33 100644 --- a/cloudinit/sources/helpers/upcloud.py +++ b/cloudinit/sources/helpers/upcloud.py @@ -111,10 +111,16 @@ def convert_to_network_config_v1(config): def _get_subnet_config(ip_addr, dns): if ip_addr.get("dhcp"): - return {"type": "dhcp6" if ip_addr.get("family") == "IPv6" else "dhcp"} - + dhcp_type = "dhcp" + if ip_addr.get("family") == "IPv6": + dhcp_type = "dhcp6" + return {"type": dhcp_type} + + static_type = "static" + if ip_addr.get("family") == "IPv6": + static_type = "static6" subpart = { - "type": "static6" if ip_addr.get("family") == "IPv6" else "static", + "type": static_type, "control": "auto", "address": ip_addr.get("address"), } @@ -156,7 +162,11 @@ def _get_subnet_config(ip_addr, dns): raw_iface.get("index"), ) - interface = {"type": "physical", "name": sysfs_name, "mac_address": mac_address} + interface = { + "type": "physical", + "name": sysfs_name, + "mac_address": mac_address + } subnets = [] for ip_address in raw_iface.get("ip_addresses"): @@ -168,7 +178,10 @@ def _get_subnet_config(ip_addr, dns): if config.get("dns"): LOG.debug("Setting DNS nameservers to %s", config.get("dns")) - nic_configs.append({"type": "nameserver", "address": config.get("dns")}) + nic_configs.append({ + "type": "nameserver", + "address": config.get("dns") + }) return {"version": 1, "config": nic_configs} @@ -194,7 +207,8 @@ def read_sysinfo(): server_uuid = dmi.read_dmi_data("system-uuid") if server_uuid: - LOG.debug("system identified via SMBIOS as UpCloud server: %s", server_uuid) + LOG.debug("system identified via SMBIOS as UpCloud server: %s", + server_uuid) else: msg = ( "system identified via SMBIOS as a UpCloud server, but " From a6df5406e83565790d8dd472da09a3751478e749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 15 Dec 2020 21:18:15 +0200 Subject: [PATCH 03/16] UpCloud datasource: unit tests --- .../unittests/test_datasource/test_common.py | 2 + .../unittests/test_datasource/test_upcloud.py | 300 ++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 tests/unittests/test_datasource/test_upcloud.py diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 4ab5d471904..0a667792fe6 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -27,6 +27,7 @@ DataSourceRbxCloud as RbxCloud, DataSourceScaleway as Scaleway, DataSourceSmartOS as SmartOS, + DataSourceUpCloud as UpCloud, ) from cloudinit.sources import DataSourceNone as DSNone @@ -63,6 +64,7 @@ NoCloud.DataSourceNoCloudNet, OpenStack.DataSourceOpenStack, OVF.DataSourceOVFNet, + UpCloud.DataSourceUpCloud, ] diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py new file mode 100644 index 00000000000..7e1ae70428f --- /dev/null +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -0,0 +1,300 @@ +# Author: Antti Myyrä +# +# This file is part of cloud-init. See LICENSE file for license information. + +import json + +from cloudinit import helpers +from cloudinit import settings +from cloudinit import sources +from cloudinit.sources.DataSourceUpCloud import DataSourceUpCloud + +from cloudinit.tests.helpers import mock, CiTestCase + +UC_METADATA = json.loads(""" +{ + "cloud_name": "upcloud", + "instance_id": "00322b68-0096-4042-9406-faad61922128", + "hostname": "test.example.com", + "platform": "servers", + "subplatform": "metadata (http://169.254.169.254)", + "public_keys": [ + "ssh-rsa AAAAB.... test1@example.com", + "ssh-rsa AAAAB.... test2@example.com" + ], + "region": "fi-hel2", + "network": { + "interfaces": [ + { + "index": 1, + "ip_addresses": [ + { + "address": "94.237.105.53", + "dhcp": true, + "dns": [ + "94.237.127.9", + "94.237.40.9" + ], + "family": "IPv4", + "floating": false, + "gateway": "94.237.104.1", + "network": "94.237.104.0/22" + }, + { + "address": "94.237.105.50", + "dhcp": false, + "dns": null, + "family": "IPv4", + "floating": true, + "gateway": "", + "network": "94.237.105.50/32" + } + ], + "mac": "3a:d6:ba:4a:36:e7", + "network_id": "031457f4-0f8c-483c-96f2-eccede02909c", + "type": "public" + }, + { + "index": 2, + "ip_addresses": [ + { + "address": "10.6.3.27", + "dhcp": true, + "dns": null, + "family": "IPv4", + "floating": false, + "gateway": "10.6.0.1", + "network": "10.6.0.0/22" + } + ], + "mac": "3a:d6:ba:4a:84:cc", + "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1", + "type": "utility" + }, + { + "index": 3, + "ip_addresses": [ + { + "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7", + "dhcp": true, + "dns": [ + "2a04:3540:53::1", + "2a04:3544:53::1" + ], + "family": "IPv6", + "floating": false, + "gateway": "2a04:3545:1000:720::1", + "network": "2a04:3545:1000:720::/64" + } + ], + "mac": "3a:d6:ba:4a:63:e7", + "network_id": "03000000-0000-4000-8046-000000000000", + "type": "public" + }, + { + "index": 4, + "ip_addresses": [ + { + "address": "172.30.1.10", + "dhcp": true, + "dns": null, + "family": "IPv4", + "floating": false, + "gateway": "172.30.1.1", + "network": "172.30.1.0/24" + } + ], + "mac": "3a:d6:ba:4a:8a:e1", + "network_id": "035a0a4a-7704-4de5-820d-189fc8132714", + "type": "private" + } + ], + "dns": [ + "94.237.127.9", + "94.237.40.9" + ] + }, + "storage": { + "disks": [ + { + "id": "014efb65-223b-4d44-8f0a-c29535b88dcf", + "serial": "014efb65223b4d448f0a", + "size": 10240, + "type": "disk", + "tier": "maxiops" + } + ] + }, + "tags": [], + "user_data": "", + "vendor_data": "" +} +""") + +UC_METADATA["user_data"] = b"""#cloud-config +runcmd: +- [touch, /root/cloud-init-worked ] +""" + +MD_URL = 'http://169.254.169.254/metadata/v1.json' + + +def _mock_dmi(): + return True, "00322b68-0096-4042-9406-faad61922128" + + +class TestUpCloudMetadata(CiTestCase): + """ + Test reading the meta-data + """ + def setUp(self): + super(TestUpCloudMetadata, self).setUp() + self.tmp = self.tmp_dir() + + def get_ds(self, get_sysinfo=_mock_dmi): + ds = DataSourceUpCloud( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + if get_sysinfo: + ds._get_sysinfo = get_sysinfo + return ds + + @mock.patch('cloudinit.sources.helpers.upcloud.read_sysinfo') + def test_returns_false_not_on_upcloud(self, m_read_sysinfo): + m_read_sysinfo.return_value = (False, None) + ds = self.get_ds(get_sysinfo=None) + self.assertEqual(False, ds.get_data()) + self.assertTrue(m_read_sysinfo.called) + + @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') + def test_metadata(self, mock_readmd): + mock_readmd.return_value = UC_METADATA.copy() + + ds = self.get_ds() + ds.perform_dhcp_setup = False + + ret = ds.get_data() + self.assertTrue(ret) + + self.assertTrue(mock_readmd.called) + + self.assertEqual(UC_METADATA.get('user_data'), ds.get_userdata_raw()) + self.assertEqual(UC_METADATA.get('vendor_data'), ds.get_vendordata_raw()) + self.assertEqual(UC_METADATA.get('region'), ds.availability_zone) + self.assertEqual(UC_METADATA.get('instance_id'), ds.get_instance_id()) + self.assertEqual(UC_METADATA.get('cloud_name'), ds.cloud_name) + + self.assertEqual(UC_METADATA.get('public_keys'), + ds.get_public_ssh_keys()) + self.assertIsInstance(ds.get_public_ssh_keys(), list) + + +class TestUpCloudNetworkSetup(CiTestCase): + """ + Test reading the meta-data on networked context + """ + + def setUp(self): + super(TestUpCloudNetworkSetup, self).setUp() + self.tmp = self.tmp_dir() + + def get_ds(self, get_sysinfo=_mock_dmi): + ds = DataSourceUpCloud( + settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) + if get_sysinfo: + ds._get_sysinfo = get_sysinfo + return ds + + @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') + @mock.patch('cloudinit.net.find_fallback_nic') + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') + def test_network_configured_metadata(self, m_net, m_dhcp, m_fallback_nic, mock_readmd): + mock_readmd.return_value = UC_METADATA.copy() + + m_fallback_nic.return_value = 'eth1' + m_dhcp.return_value = [{ + 'interface': 'eth1', 'fixed-address': '10.6.3.27', + 'routers': '10.6.0.1', 'subnet-mask': '22', + 'broadcast-address': '10.6.3.255'} + ] + + ds = self.get_ds() + + ret = ds.get_data() + self.assertTrue(ret) + + self.assertTrue(m_dhcp.called) + m_dhcp.assert_called_with('eth1', None) + + m_net.assert_called_once_with( + broadcast='10.6.3.255', interface='eth1', + ip='10.6.3.27', prefix_or_mask='22', + router='10.6.0.1', static_routes=None + ) + + self.assertTrue(mock_readmd.called) + + self.assertEqual(UC_METADATA.get('region'), ds.availability_zone) + self.assertEqual(UC_METADATA.get('instance_id'), ds.get_instance_id()) + self.assertEqual(UC_METADATA.get('cloud_name'), ds.cloud_name) + + @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_network_configuration(self, m_get_by_mac, mock_readmd): + mock_readmd.return_value = UC_METADATA.copy() + + raw_ifaces = UC_METADATA.get('network').get('interfaces') + self.assertEqual(4, len(raw_ifaces)) + + m_get_by_mac.return_value = { + raw_ifaces[0].get('mac'): 'eth0', + raw_ifaces[1].get('mac'): 'eth1', + raw_ifaces[2].get('mac'): 'eth2', + raw_ifaces[3].get('mac'): 'eth3', + } + + ds = self.get_ds() + ds.perform_dhcp_setup = False + + ret = ds.get_data() + self.assertTrue(ret) + + self.assertTrue(mock_readmd.called) + + netcfg = ds.network_config + print(netcfg) + + self.assertEqual(1, netcfg.get('version')) + + config = netcfg.get('config') + self.assertIsInstance(config, list) + self.assertEqual(5, len(config)) + self.assertEqual('physical', config[3].get('type')) + + self.assertEqual(raw_ifaces[2].get('mac'), config[2].get('mac_address')) + self.assertEqual(1, len(config[2].get('subnets'))) + self.assertEqual('dhcp6', config[2].get('subnets')[0].get('type')) + + self.assertEqual(2, len(config[0].get('subnets'))) + self.assertEqual('static', config[0].get('subnets')[1].get('type')) + + dns = config[4] + self.assertEqual('nameserver', dns.get('type')) + self.assertEqual(2, len(dns.get('address'))) + self.assertEqual(UC_METADATA.get('network').get('dns')[1], dns.get('address')[1]) + + +class TestUpCloudDatasourceLoading(CiTestCase): + def test_get_datasource_list_returns_in_local(self): + deps = (sources.DEP_FILESYSTEM, sources.DEP_NETWORK) + ds_list = sources.DataSourceUpCloud.get_datasource_list(deps) + self.assertEqual(ds_list, + [DataSourceUpCloud]) + + def test_list_sources_finds_ds(self): + found = sources.list_sources( + ['UpCloud'], (sources.DEP_FILESYSTEM, sources.DEP_NETWORK), ['cloudinit.sources']) + self.assertEqual([DataSourceUpCloud], + found) + +# vi: ts=4 expandtab From f6c3254433cf1a970de3ee0a940c86001f36030a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Mon, 28 Dec 2020 15:20:50 +0200 Subject: [PATCH 04/16] UpCloud datasource: support both local and networked init --- cloudinit/sources/DataSourceUpCloud.py | 24 +++++++++++++++---- .../unittests/test_datasource/test_common.py | 1 + .../unittests/test_datasource/test_upcloud.py | 10 ++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index 3051d71f883..2e28a9b8515 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -20,7 +20,7 @@ # Wait for a up to a minute, retrying the meta-data server # every 2 seconds. -MD_RETRIES = 30 +MD_RETRIES = 5 MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 @@ -29,6 +29,9 @@ class DataSourceUpCloud(sources.DataSource): dsname = "UpCloud" + # We'll perform DHCP setup only in init-local, see DataSourceUpCloudLocal + perform_dhcp_setup = False + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.distro = distro @@ -43,7 +46,6 @@ def __init__(self, sys_cfg, distro, paths): self.retries = self.ds_cfg.get("retries", MD_RETRIES) self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT) self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY) - self.perform_dhcp_setup = self.ds_cfg.get("dhcp_setup", True) self._network_config = None def _get_sysinfo(self): @@ -133,9 +135,23 @@ def network_config(self): return self._network_config +class DataSourceUpCloudLocal(DataSourceUpCloud): + """Run in init-local using a DHCP discovery prior to metadata crawl. + + In init-local, no network is available. This subclass sets up minimal + networking with dhclient on a viable nic so that it can talk to the + metadata service. If the metadata service provides network configuration + then render the network configuration for that instance based on metadata. + """ + + perform_dhcp_setup = True # Get metadata network config if present + + # Used to match classes to dependencies -datasources = [(DataSourceUpCloud, - (sources.DEP_FILESYSTEM, sources.DEP_NETWORK))] +datasources = [ + (DataSourceUpCloudLocal, (sources.DEP_FILESYSTEM, )), + (DataSourceUpCloud, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] # Return a list of data sources that match this set of dependencies diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 0a667792fe6..5912f7eeb1d 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -49,6 +49,7 @@ OpenStack.DataSourceOpenStackLocal, RbxCloud.DataSourceRbxCloud, Scaleway.DataSourceScaleway, + UpCloud.DataSourceUpCloudLocal, ] DEFAULT_NETWORK = [ diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index 7e1ae70428f..e24cf8e0558 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -7,7 +7,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit import sources -from cloudinit.sources.DataSourceUpCloud import DataSourceUpCloud +from cloudinit.sources.DataSourceUpCloud import DataSourceUpCloud, DataSourceUpCloudLocal from cloudinit.tests.helpers import mock, CiTestCase @@ -198,7 +198,7 @@ def setUp(self): self.tmp = self.tmp_dir() def get_ds(self, get_sysinfo=_mock_dmi): - ds = DataSourceUpCloud( + ds = DataSourceUpCloudLocal( settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})) if get_sysinfo: ds._get_sysinfo = get_sysinfo @@ -286,6 +286,12 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): class TestUpCloudDatasourceLoading(CiTestCase): def test_get_datasource_list_returns_in_local(self): + deps = (sources.DEP_FILESYSTEM, ) + ds_list = sources.DataSourceUpCloud.get_datasource_list(deps) + self.assertEqual(ds_list, + [DataSourceUpCloudLocal]) + + def test_get_datasource_list_returns_in_normal(self): deps = (sources.DEP_FILESYSTEM, sources.DEP_NETWORK) ds_list = sources.DataSourceUpCloud.get_datasource_list(deps) self.assertEqual(ds_list, From 3d77ab835dcca772dd61ab06d38067a71dd9fcb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Mon, 28 Dec 2020 15:53:33 +0200 Subject: [PATCH 05/16] UpCloud datasource: Use SLAAC config for IPv6 interfaces --- cloudinit/sources/helpers/upcloud.py | 4 +++- tests/unittests/test_datasource/test_upcloud.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py index 5d287ebde33..b38e09cb6bc 100644 --- a/cloudinit/sources/helpers/upcloud.py +++ b/cloudinit/sources/helpers/upcloud.py @@ -113,7 +113,9 @@ def _get_subnet_config(ip_addr, dns): if ip_addr.get("dhcp"): dhcp_type = "dhcp" if ip_addr.get("family") == "IPv6": - dhcp_type = "dhcp6" + # UpCloud currently passes IPv6 addresses via + # StateLess Address Auto Configuration (SLAAC) + dhcp_type = "ipv6_dhcpv6-stateless" return {"type": dhcp_type} static_type = "static" diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index e24cf8e0558..92fbfb90f4c 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -273,7 +273,7 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): self.assertEqual(raw_ifaces[2].get('mac'), config[2].get('mac_address')) self.assertEqual(1, len(config[2].get('subnets'))) - self.assertEqual('dhcp6', config[2].get('subnets')[0].get('type')) + self.assertEqual('ipv6_dhcpv6-stateless', config[2].get('subnets')[0].get('type')) self.assertEqual(2, len(config[0].get('subnets'))) self.assertEqual('static', config[0].get('subnets')[1].get('type')) From 0bb8a992a2b1252bd9a2b06703f553a6595e4f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Mon, 28 Dec 2020 17:11:33 +0200 Subject: [PATCH 06/16] Documentation for UpCloud datasource --- doc/rtd/topics/datasources.rst | 1 + doc/rtd/topics/datasources/upcloud.rst | 33 ++++++++++++++++++++++++++ doc/rtd/topics/network-config.rst | 5 ++++ 3 files changed, 39 insertions(+) create mode 100644 doc/rtd/topics/datasources/upcloud.rst diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 3d026143b94..228173d25d8 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -47,6 +47,7 @@ The following is a list of documents for each supported datasource: datasources/ovf.rst datasources/rbxcloud.rst datasources/smartos.rst + datasources/upcloud.rst datasources/zstack.rst diff --git a/doc/rtd/topics/datasources/upcloud.rst b/doc/rtd/topics/datasources/upcloud.rst new file mode 100644 index 00000000000..5d6ab70995d --- /dev/null +++ b/doc/rtd/topics/datasources/upcloud.rst @@ -0,0 +1,33 @@ +.. _datasource_upcloud: + +UpCloud +============= + +The `UpCloud`_ datasource consumes information from UpCloud's `metadata +service`_. This metadata service serves information about the +running server via HTTP over the address 169.254.169.254 available in every +DHCP-configured interface. The metadata API endpoints are fully described in +UpCloud documentation at +`https://developers.upcloud.com/1.3/8-servers/#metadata-service +`_. + +Configuration +------------- + +UpCloud's datasource can be configured as follows: + + datasource: + UpCloud: + retries: 5 + timeout: 2 + +- *retries*: Determines the number of times to attempt to connect to the + metadata service +- *timeout*: Determines the timeout in seconds to wait for a response from the + metadata service + +.. _UpCloud: https://upcloud.com/ +.. _metadata service: https://developers.upcloud.com/1.3/8-servers/#metadata-service +.. _Full documentation: https://developers.upcloud.com/1.3/8-servers/#metadata-service + +.. vi: textwidth=78 diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index 08db04d89c5..07cad765443 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -144,6 +144,10 @@ The following Datasources optionally provide network configuration: - `SmartOS JSON Metadata`_ +- :ref:`datasource_upcloud` + + - `UpCloud JSON metadata`_ + For more information on network configuration formats .. toctree:: @@ -257,5 +261,6 @@ Example output converting V2 to sysconfig: .. _DigitalOcean JSON metadata: https://developers.digitalocean.com/documentation/metadata/#network-interfaces-index .. _OpenStack Metadata Service Network: https://specs.openstack.org/openstack/nova-specs/specs/liberty/implemented/metadata-service-network-info.html .. _SmartOS JSON Metadata: https://eng.joyent.com/mdata/datadict.html +.. _UpCloud JSON metadata: https://developers.upcloud.com/1.3/8-servers/#metadata-service .. vi: textwidth=78 From d9166b0dc9ce39c9ec05992dd5b06bf515deb610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Mon, 28 Dec 2020 17:31:03 +0200 Subject: [PATCH 07/16] UpCloud datasource: required styling for tests --- .../unittests/test_datasource/test_upcloud.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index 92fbfb90f4c..c160c445eda 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -7,7 +7,8 @@ from cloudinit import helpers from cloudinit import settings from cloudinit import sources -from cloudinit.sources.DataSourceUpCloud import DataSourceUpCloud, DataSourceUpCloudLocal +from cloudinit.sources.DataSourceUpCloud import DataSourceUpCloud, \ + DataSourceUpCloudLocal from cloudinit.tests.helpers import mock, CiTestCase @@ -178,7 +179,8 @@ def test_metadata(self, mock_readmd): self.assertTrue(mock_readmd.called) self.assertEqual(UC_METADATA.get('user_data'), ds.get_userdata_raw()) - self.assertEqual(UC_METADATA.get('vendor_data'), ds.get_vendordata_raw()) + self.assertEqual(UC_METADATA.get('vendor_data'), + ds.get_vendordata_raw()) self.assertEqual(UC_METADATA.get('region'), ds.availability_zone) self.assertEqual(UC_METADATA.get('instance_id'), ds.get_instance_id()) self.assertEqual(UC_METADATA.get('cloud_name'), ds.cloud_name) @@ -208,7 +210,8 @@ def get_ds(self, get_sysinfo=_mock_dmi): @mock.patch('cloudinit.net.find_fallback_nic') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') - def test_network_configured_metadata(self, m_net, m_dhcp, m_fallback_nic, mock_readmd): + def test_network_configured_metadata(self, m_net, m_dhcp, + m_fallback_nic, mock_readmd): mock_readmd.return_value = UC_METADATA.copy() m_fallback_nic.return_value = 'eth1' @@ -271,9 +274,11 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): self.assertEqual(5, len(config)) self.assertEqual('physical', config[3].get('type')) - self.assertEqual(raw_ifaces[2].get('mac'), config[2].get('mac_address')) + self.assertEqual(raw_ifaces[2].get('mac'), config[2] + .get('mac_address')) self.assertEqual(1, len(config[2].get('subnets'))) - self.assertEqual('ipv6_dhcpv6-stateless', config[2].get('subnets')[0].get('type')) + self.assertEqual('ipv6_dhcpv6-stateless', config[2].get('subnets')[0] + .get('type')) self.assertEqual(2, len(config[0].get('subnets'))) self.assertEqual('static', config[0].get('subnets')[1].get('type')) @@ -281,7 +286,8 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): dns = config[4] self.assertEqual('nameserver', dns.get('type')) self.assertEqual(2, len(dns.get('address'))) - self.assertEqual(UC_METADATA.get('network').get('dns')[1], dns.get('address')[1]) + self.assertEqual(UC_METADATA.get('network').get('dns')[1], + dns.get('address')[1]) class TestUpCloudDatasourceLoading(CiTestCase): @@ -299,7 +305,8 @@ def test_get_datasource_list_returns_in_normal(self): def test_list_sources_finds_ds(self): found = sources.list_sources( - ['UpCloud'], (sources.DEP_FILESYSTEM, sources.DEP_NETWORK), ['cloudinit.sources']) + ['UpCloud'], (sources.DEP_FILESYSTEM, sources.DEP_NETWORK), + ['cloudinit.sources']) self.assertEqual([DataSourceUpCloud], found) From e5b434ca88166dca3f0c5709c5e365ab1589c3a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 19 Jan 2021 16:35:29 +0200 Subject: [PATCH 08/16] Documentation addition/fix --- doc/rtd/topics/availability.rst | 1 + doc/rtd/topics/datasources/upcloud.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index 8f56a7d27f9..f58b2b388e8 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -55,6 +55,7 @@ environments in the public cloud: - CloudStack - AltCloud - SmartOS +- UpCloud Additionally, cloud-init is supported on these private clouds: diff --git a/doc/rtd/topics/datasources/upcloud.rst b/doc/rtd/topics/datasources/upcloud.rst index 5d6ab70995d..ecba7c2a067 100644 --- a/doc/rtd/topics/datasources/upcloud.rst +++ b/doc/rtd/topics/datasources/upcloud.rst @@ -28,6 +28,6 @@ UpCloud's datasource can be configured as follows: .. _UpCloud: https://upcloud.com/ .. _metadata service: https://developers.upcloud.com/1.3/8-servers/#metadata-service -.. _Full documentation: https://developers.upcloud.com/1.3/8-servers/#metadata-service +.. _Full documentation: https://upcloud.com/community/tutorials/upcloud-metadata-service/ .. vi: textwidth=78 From f6b06dd13bfe22f93f34761b3ce79f199c7cdcaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 19 Jan 2021 16:46:21 +0200 Subject: [PATCH 09/16] UpCloud to ds-identity --- tools/ds-identify | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/ds-identify b/tools/ds-identify index 496dbb8a727..2f2486f76a3 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -125,7 +125,7 @@ DI_DSNAME="" # be searched if there is no setting found in config. DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ -OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud" +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -883,6 +883,11 @@ dscheck_RbxCloud() { return ${DS_NOT_FOUND} } +dscheck_UpCloud() { + dmi_sys_vendor_is UpCloud && return ${DS_FOUND} + return ${DS_NOT_FOUND} +} + ovf_vmware_guest_customization() { # vmware guest customization From 4046d404e4fa779153793b5228df4c808de2145d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 19 Jan 2021 16:48:48 +0200 Subject: [PATCH 10/16] UpCloud datasource default to v2 network config --- cloudinit/sources/DataSourceUpCloud.py | 19 +- cloudinit/sources/helpers/upcloud.py | 289 ++++++++++++------ .../unittests/test_datasource/test_upcloud.py | 47 ++- 3 files changed, 252 insertions(+), 103 deletions(-) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index 2e28a9b8515..87754895b5e 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -23,6 +23,7 @@ MD_RETRIES = 5 MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 +MD_NETWORK_CONFIG_VERSION = 2 class DataSourceUpCloud(sources.DataSource): @@ -46,6 +47,9 @@ def __init__(self, sys_cfg, distro, paths): self.retries = self.ds_cfg.get("retries", MD_RETRIES) self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT) self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY) + self.network_config_version = self.ds_cfg.get( + "network_config_version", MD_NETWORK_CONFIG_VERSION + ) self._network_config = None def _get_sysinfo(self): @@ -117,9 +121,10 @@ def check_instance_id(self, sys_cfg): @property def network_config(self): - """Configure the networking. This needs to be done each boot, - since the IP and interface information might have changed - due to reconfiguration. + """ + Configure the networking. This needs to be done each boot, + since the IP and interface information might have changed + due to reconfiguration. """ if self._network_config: @@ -129,14 +134,16 @@ def network_config(self): if not raw_network_config: raise Exception("Unable to get network meta-data from server....") - self._network_config = uc_helper.convert_to_network_config_v1( - raw_network_config + self._network_config = uc_helper.convert_network_config( + raw_network_config, self.network_config_version ) + return self._network_config class DataSourceUpCloudLocal(DataSourceUpCloud): - """Run in init-local using a DHCP discovery prior to metadata crawl. + """ + Run in init-local using a DHCP discovery prior to metadata crawl. In init-local, no network is available. This subclass sets up minimal networking with dhclient on a viable nic so that it can talk to the diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py index b38e09cb6bc..53fa98e3285 100644 --- a/cloudinit/sources/helpers/upcloud.py +++ b/cloudinit/sources/helpers/upcloud.py @@ -12,101 +12,189 @@ LOG = logging.getLogger(__name__) +def convert_to_network_config_v2(config): + """ + Convert the UpCloud network metadata description into + Cloud-init's version 2 netconfig format. + + Example JSON: + { + "interfaces": [ + { + "index": 1, + "ip_addresses": [ + { + "address": "94.237.105.53", + "dhcp": true, + "dns": [ + "94.237.127.9", + "94.237.40.9" + ], + "family": "IPv4", + "floating": false, + "gateway": "94.237.104.1", + "network": "94.237.104.0/22" + }, + { + "address": "94.237.105.50", + "dhcp": false, + "dns": [], + "family": "IPv4", + "floating": true, + "gateway": "", + "network": "94.237.105.50/32" + } + ], + "mac": "32:d5:ba:4a:36:e7", + "network_id": "031457f4-0f8c-483c-96f2-eccede02909c", + "type": "public" + }, + { + "index": 2, + "ip_addresses": [ + { + "address": "10.6.3.27", + "dhcp": true, + "dns": [], + "family": "IPv4", + "floating": false, + "gateway": "10.6.0.1", + "network": "10.6.0.0/22" + } + ], + "mac": "32:d5:ba:4a:84:cc", + "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1", + "type": "utility" + }, + { + "index": 3, + "ip_addresses": [ + { + "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7", + "dhcp": true, + "dns": [ + "2a04:3540:53::1", + "2a04:3544:53::1" + ], + "family": "IPv6", + "floating": false, + "gateway": "2a04:3545:1000:720::1", + "network": "2a04:3545:1000:720::/64" + } + ], + "mac": "32:d5:ba:4a:63:e7", + "network_id": "03000000-0000-4000-8046-000000000000", + "type": "public" + }, + { + "index": 4, + "ip_addresses": [ + { + "address": "172.30.1.10", + "dhcp": true, + "dns": [], + "family": "IPv4", + "floating": false, + "gateway": "172.30.1.1", + "network": "172.30.1.0/24" + } + ], + "mac": "32:d5:ba:4a:8a:e1", + "network_id": "035a0a4a-77b4-4de5-820d-189fc8135714", + "type": "private" + } + ], + "dns": [ + "94.237.127.9", + "94.237.40.9" + ] + } + """ + + nic_configs = {} + macs_to_interfaces = cloudnet.get_interfaces_by_mac() + LOG.debug("NIC mapping: %s", macs_to_interfaces) + for raw_iface in config.get("interfaces"): + LOG.debug("Considering %s", raw_iface) + + mac_address = raw_iface.get("mac") + if mac_address not in macs_to_interfaces: + raise RuntimeError( + "Did not find network interface on system " + "with mac '%s'. Cannot apply configuration: %s" + % (mac_address, raw_iface) + ) + + iface_type = raw_iface.get("type") + sysfs_name = macs_to_interfaces.get(mac_address) + + LOG.debug( + "Found %s interface '%s' with address '%s' (index %d)", + iface_type, + sysfs_name, + mac_address, + raw_iface.get("index"), + ) + + interface = {"match": {"macaddress": mac_address}} + + iface_dns = config.get("dns") + dhcp_configured = False + + for ip_addr in raw_iface.get("ip_addresses"): + if ip_addr.get("dhcp"): + if ip_addr.get("family") == "IPv6": + interface["dhcp6"] = True + interface["accept_ra"] = True + if "dns" in ip_addr: + iface_dns = ip_addr.get("dns") + else: + interface["dhcp4"] = True + + dhcp_configured = True + continue + + if "/" not in ip_addr.get("network"): + LOG.error( + "CIDR notation missing from network, not adding %s", + ip_addr.get("address"), + ) + continue + static_address = "{}/{}".format( + ip_addr.get("address"), ip_addr.get("network").split("/")[1] + ) + + if "addresses" not in interface: + interface["addresses"] = [] + interface["addresses"].append(static_address) + + if ip_addr.get("gateway"): + interface[ + "gateway{}".format(6 if "family" == "IPv6" else 4) + ] = ip_addr.get("gateway") + + if ip_addr.get("dns") and\ + (ip_addr.get("dns") != config.get("dns")): + interface["nameservers"] = ip_addr.get("dns") + + if not dhcp_configured: + interface["nameservers"] = {"addresses": iface_dns} + + LOG.debug( + "Generated interface config for %s: %s", + sysfs_name, interface + ) + nic_configs[sysfs_name] = interface + + return {"version": 2, "ethernets": nic_configs} + + def convert_to_network_config_v1(config): - """Convert the UpCloud network metadata description into - Cloud-init's version 1 netconfig format. - - Example JSON: - "interfaces": [ - { - "index": 1, - "ip_addresses": [ - { - "address": "94.237.105.53", - "dhcp": true, - "dns": [ - "94.237.127.9", - "94.237.40.9" - ], - "family": "IPv4", - "floating": false, - "gateway": "94.237.104.1", - "network": "94.237.104.0/22" - }, - { - "address": "94.237.105.50", - "dhcp": false, - "dns": [], - "family": "IPv4", - "floating": true, - "gateway": "", - "network": "94.237.105.50/32" - } - ], - "mac": "32:d5:ba:4a:36:e7", - "network_id": "031457f4-0f8c-483c-96f2-eccede02909c", - "type": "public" - }, - { - "index": 2, - "ip_addresses": [ - { - "address": "10.6.3.27", - "dhcp": true, - "dns": [], - "family": "IPv4", - "floating": false, - "gateway": "10.6.0.1", - "network": "10.6.0.0/22" - } - ], - "mac": "32:d5:ba:4a:84:cc", - "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1", - "type": "utility" - }, - { - "index": 3, - "ip_addresses": [ - { - "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7", - "dhcp": true, - "dns": [ - "2a04:3540:53::1", - "2a04:3544:53::1" - ], - "family": "IPv6", - "floating": false, - "gateway": "2a04:3545:1000:720::1", - "network": "2a04:3545:1000:720::/64" - } - ], - "mac": "32:d5:ba:4a:63:e7", - "network_id": "03000000-0000-4000-8046-000000000000", - "type": "public" - }, - { - "index": 4, - "ip_addresses": [ - { - "address": "172.30.1.10", - "dhcp": true, - "dns": [], - "family": "IPv4", - "floating": false, - "gateway": "172.30.1.1", - "network": "172.30.1.0/24" - } - ], - "mac": "32:d5:ba:4a:8a:e1", - "network_id": "035a0a4a-77b4-4de5-820d-189fc8135714", - "type": "private" - } - ], - "dns": [ - "94.237.127.9", - "94.237.40.9" - ] - } + """ + Convert the UpCloud network metadata description into + Cloud-init's version 1 netconfig format. + + Example JSON can be seen above in v2 conversion function. """ def _get_subnet_config(ip_addr, dns): @@ -188,6 +276,13 @@ def _get_subnet_config(ip_addr, dns): return {"version": 1, "config": nic_configs} +def convert_network_config(config, version=2): + if version == 2: + return convert_to_network_config_v2(config) + + return convert_to_network_config_v1(config) + + def read_metadata(url, timeout=2, sec_between=2, retries=30): response = url_helper.readurl( url, timeout=timeout, sec_between=sec_between, retries=retries @@ -209,8 +304,10 @@ def read_sysinfo(): server_uuid = dmi.read_dmi_data("system-uuid") if server_uuid: - LOG.debug("system identified via SMBIOS as UpCloud server: %s", - server_uuid) + LOG.debug( + "system identified via SMBIOS as UpCloud server: %s", + server_uuid + ) else: msg = ( "system identified via SMBIOS as a UpCloud server, but " diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index c160c445eda..835dae3cc5f 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -265,7 +265,52 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): self.assertTrue(mock_readmd.called) netcfg = ds.network_config - print(netcfg) + + self.assertEqual(2, netcfg.get('version')) + + ethernets = netcfg.get('ethernets') + self.assertIsInstance(ethernets, dict) + self.assertEqual(4, len(ethernets.keys())) + self.assertTrue('dhcp4' in ethernets.get('eth0')) + self.assertTrue(ethernets.get('eth0').get('dhcp4')) + + self.assertEqual(raw_ifaces[2].get('mac'), ethernets.get('eth2').get('match').get('macaddress')) + self.assertEqual(1, len(ethernets.get('eth0').get('addresses'))) + + self.assertTrue('dhcp4' not in ethernets.get('eth2')) + self.assertTrue('dhcp6' in ethernets.get('eth2')) + self.assertTrue(ethernets.get('eth2').get('dhcp6')) + self.assertTrue('accept_ra' in ethernets.get('eth2')) + self.assertTrue(ethernets.get('eth2').get('accept_ra')) + + + @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_v1_network_configuration(self, m_get_by_mac, mock_readmd): + mock_readmd.return_value = UC_METADATA.copy() + + raw_ifaces = UC_METADATA.get('network').get('interfaces') + self.assertEqual(4, len(raw_ifaces)) + + m_get_by_mac.return_value = { + raw_ifaces[0].get('mac'): 'eth0', + raw_ifaces[1].get('mac'): 'eth1', + raw_ifaces[2].get('mac'): 'eth2', + raw_ifaces[3].get('mac'): 'eth3', + } + + ds = self.get_ds() + ds.perform_dhcp_setup = False + + # Version 2 is the defaults, so we're overriding the config option. + ds.network_config_version = 1 + + ret = ds.get_data() + self.assertTrue(ret) + + self.assertTrue(mock_readmd.called) + + netcfg = ds.network_config self.assertEqual(1, netcfg.get('version')) From 7cb9f7697e4a0f82e11a431d2f9b696354b95ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 19 Jan 2021 17:21:47 +0200 Subject: [PATCH 11/16] Flake: forgotten styling --- tests/unittests/test_datasource/test_upcloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index 835dae3cc5f..043011e455f 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -274,7 +274,10 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): self.assertTrue('dhcp4' in ethernets.get('eth0')) self.assertTrue(ethernets.get('eth0').get('dhcp4')) - self.assertEqual(raw_ifaces[2].get('mac'), ethernets.get('eth2').get('match').get('macaddress')) + self.assertEqual( + raw_ifaces[2].get('mac'), + ethernets.get('eth2').get('match').get('macaddress') + ) self.assertEqual(1, len(ethernets.get('eth0').get('addresses'))) self.assertTrue('dhcp4' not in ethernets.get('eth2')) @@ -283,7 +286,6 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): self.assertTrue('accept_ra' in ethernets.get('eth2')) self.assertTrue(ethernets.get('eth2').get('accept_ra')) - @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') @mock.patch('cloudinit.net.get_interfaces_by_mac') def test_v1_network_configuration(self, m_get_by_mac, mock_readmd): From 03aa632b54f02340be8a7df0b23a1cfe1ae5b66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Thu, 28 Jan 2021 16:04:10 +0200 Subject: [PATCH 12/16] UpCloud datasource: revert to net config v1, fix accept-ra in v2 --- cloudinit/sources/DataSourceUpCloud.py | 2 +- cloudinit/sources/helpers/upcloud.py | 2 +- .../unittests/test_datasource/test_upcloud.py | 73 ++++++++++--------- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index 87754895b5e..6e9c42ca9b6 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -23,7 +23,7 @@ MD_RETRIES = 5 MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 -MD_NETWORK_CONFIG_VERSION = 2 +MD_NETWORK_CONFIG_VERSION = 1 class DataSourceUpCloud(sources.DataSource): diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py index 53fa98e3285..8e15496c1ee 100644 --- a/cloudinit/sources/helpers/upcloud.py +++ b/cloudinit/sources/helpers/upcloud.py @@ -145,7 +145,7 @@ def convert_to_network_config_v2(config): if ip_addr.get("dhcp"): if ip_addr.get("family") == "IPv6": interface["dhcp6"] = True - interface["accept_ra"] = True + interface["accept-ra"] = True if "dns" in ip_addr: iface_dns = ip_addr.get("dns") else: diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index 043011e455f..2228281d484 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -266,25 +266,28 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): netcfg = ds.network_config - self.assertEqual(2, netcfg.get('version')) + self.assertEqual(1, netcfg.get('version')) - ethernets = netcfg.get('ethernets') - self.assertIsInstance(ethernets, dict) - self.assertEqual(4, len(ethernets.keys())) - self.assertTrue('dhcp4' in ethernets.get('eth0')) - self.assertTrue(ethernets.get('eth0').get('dhcp4')) + config = netcfg.get('config') + self.assertIsInstance(config, list) + self.assertEqual(5, len(config)) + self.assertEqual('physical', config[3].get('type')) - self.assertEqual( - raw_ifaces[2].get('mac'), - ethernets.get('eth2').get('match').get('macaddress') - ) - self.assertEqual(1, len(ethernets.get('eth0').get('addresses'))) + self.assertEqual(raw_ifaces[2].get('mac'), config[2] + .get('mac_address')) + self.assertEqual(1, len(config[2].get('subnets'))) + self.assertEqual('ipv6_dhcpv6-stateless', config[2].get('subnets')[0] + .get('type')) + + self.assertEqual(2, len(config[0].get('subnets'))) + self.assertEqual('static', config[0].get('subnets')[1].get('type')) + + dns = config[4] + self.assertEqual('nameserver', dns.get('type')) + self.assertEqual(2, len(dns.get('address'))) + self.assertEqual(UC_METADATA.get('network').get('dns')[1], + dns.get('address')[1]) - self.assertTrue('dhcp4' not in ethernets.get('eth2')) - self.assertTrue('dhcp6' in ethernets.get('eth2')) - self.assertTrue(ethernets.get('eth2').get('dhcp6')) - self.assertTrue('accept_ra' in ethernets.get('eth2')) - self.assertTrue(ethernets.get('eth2').get('accept_ra')) @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') @mock.patch('cloudinit.net.get_interfaces_by_mac') @@ -304,8 +307,8 @@ def test_v1_network_configuration(self, m_get_by_mac, mock_readmd): ds = self.get_ds() ds.perform_dhcp_setup = False - # Version 2 is the defaults, so we're overriding the config option. - ds.network_config_version = 1 + # Version 1 is the default, so we're overriding the config option. + ds.network_config_version = 2 ret = ds.get_data() self.assertTrue(ret) @@ -314,27 +317,25 @@ def test_v1_network_configuration(self, m_get_by_mac, mock_readmd): netcfg = ds.network_config - self.assertEqual(1, netcfg.get('version')) - - config = netcfg.get('config') - self.assertIsInstance(config, list) - self.assertEqual(5, len(config)) - self.assertEqual('physical', config[3].get('type')) + self.assertEqual(2, netcfg.get('version')) - self.assertEqual(raw_ifaces[2].get('mac'), config[2] - .get('mac_address')) - self.assertEqual(1, len(config[2].get('subnets'))) - self.assertEqual('ipv6_dhcpv6-stateless', config[2].get('subnets')[0] - .get('type')) + ethernets = netcfg.get('ethernets') + self.assertIsInstance(ethernets, dict) + self.assertEqual(4, len(ethernets.keys())) + self.assertTrue('dhcp4' in ethernets.get('eth0')) + self.assertTrue(ethernets.get('eth0').get('dhcp4')) - self.assertEqual(2, len(config[0].get('subnets'))) - self.assertEqual('static', config[0].get('subnets')[1].get('type')) + self.assertEqual( + raw_ifaces[2].get('mac'), + ethernets.get('eth2').get('match').get('macaddress') + ) + self.assertEqual(1, len(ethernets.get('eth0').get('addresses'))) - dns = config[4] - self.assertEqual('nameserver', dns.get('type')) - self.assertEqual(2, len(dns.get('address'))) - self.assertEqual(UC_METADATA.get('network').get('dns')[1], - dns.get('address')[1]) + self.assertTrue('dhcp4' not in ethernets.get('eth2')) + self.assertTrue('dhcp6' in ethernets.get('eth2')) + self.assertTrue(ethernets.get('eth2').get('dhcp6')) + self.assertTrue('accept-ra' in ethernets.get('eth2')) + self.assertTrue(ethernets.get('eth2').get('accept-ra')) class TestUpCloudDatasourceLoading(CiTestCase): From 327927f0c271695d2c5e5bc17773e77f7379d42c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Thu, 28 Jan 2021 16:27:46 +0200 Subject: [PATCH 13/16] Flake: style change --- tests/unittests/test_datasource/test_upcloud.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index 2228281d484..1963add0406 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -285,9 +285,10 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): dns = config[4] self.assertEqual('nameserver', dns.get('type')) self.assertEqual(2, len(dns.get('address'))) - self.assertEqual(UC_METADATA.get('network').get('dns')[1], - dns.get('address')[1]) - + self.assertEqual( + UC_METADATA.get('network').get('dns')[1], + dns.get('address')[1] + ) @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') @mock.patch('cloudinit.net.get_interfaces_by_mac') From b0de83454ca6b67f157ecc26684a868b84a5a92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Tue, 2 Feb 2021 17:28:23 +0200 Subject: [PATCH 14/16] Network config: remove v2 --- cloudinit/sources/DataSourceUpCloud.py | 6 +- cloudinit/sources/helpers/upcloud.py | 259 ++++++------------ .../unittests/test_datasource/test_upcloud.py | 48 ---- 3 files changed, 86 insertions(+), 227 deletions(-) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index 6e9c42ca9b6..11b5857f2b8 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -23,7 +23,6 @@ MD_RETRIES = 5 MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 -MD_NETWORK_CONFIG_VERSION = 1 class DataSourceUpCloud(sources.DataSource): @@ -47,9 +46,6 @@ def __init__(self, sys_cfg, distro, paths): self.retries = self.ds_cfg.get("retries", MD_RETRIES) self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT) self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY) - self.network_config_version = self.ds_cfg.get( - "network_config_version", MD_NETWORK_CONFIG_VERSION - ) self._network_config = None def _get_sysinfo(self): @@ -135,7 +131,7 @@ def network_config(self): raise Exception("Unable to get network meta-data from server....") self._network_config = uc_helper.convert_network_config( - raw_network_config, self.network_config_version + raw_network_config, ) return self._network_config diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py index 8e15496c1ee..199baa58ad7 100644 --- a/cloudinit/sources/helpers/upcloud.py +++ b/cloudinit/sources/helpers/upcloud.py @@ -12,189 +12,103 @@ LOG = logging.getLogger(__name__) -def convert_to_network_config_v2(config): +def convert_to_network_config_v1(config): """ - Convert the UpCloud network metadata description into - Cloud-init's version 2 netconfig format. + Convert the UpCloud network metadata description into + Cloud-init's version 1 netconfig format. - Example JSON: + Example JSON: + { + "interfaces": [ { - "interfaces": [ + "index": 1, + "ip_addresses": [ { - "index": 1, - "ip_addresses": [ - { - "address": "94.237.105.53", - "dhcp": true, - "dns": [ - "94.237.127.9", - "94.237.40.9" - ], - "family": "IPv4", - "floating": false, - "gateway": "94.237.104.1", - "network": "94.237.104.0/22" - }, - { - "address": "94.237.105.50", - "dhcp": false, - "dns": [], - "family": "IPv4", - "floating": true, - "gateway": "", - "network": "94.237.105.50/32" - } + "address": "94.237.105.53", + "dhcp": true, + "dns": [ + "94.237.127.9", + "94.237.40.9" ], - "mac": "32:d5:ba:4a:36:e7", - "network_id": "031457f4-0f8c-483c-96f2-eccede02909c", - "type": "public" + "family": "IPv4", + "floating": false, + "gateway": "94.237.104.1", + "network": "94.237.104.0/22" }, { - "index": 2, - "ip_addresses": [ - { - "address": "10.6.3.27", - "dhcp": true, - "dns": [], - "family": "IPv4", - "floating": false, - "gateway": "10.6.0.1", - "network": "10.6.0.0/22" - } - ], - "mac": "32:d5:ba:4a:84:cc", - "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1", - "type": "utility" - }, + "address": "94.237.105.50", + "dhcp": false, + "dns": [], + "family": "IPv4", + "floating": true, + "gateway": "", + "network": "94.237.105.50/32" + } + ], + "mac": "32:d5:ba:4a:36:e7", + "network_id": "031457f4-0f8c-483c-96f2-eccede02909c", + "type": "public" + }, + { + "index": 2, + "ip_addresses": [ { - "index": 3, - "ip_addresses": [ - { - "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7", - "dhcp": true, - "dns": [ - "2a04:3540:53::1", - "2a04:3544:53::1" - ], - "family": "IPv6", - "floating": false, - "gateway": "2a04:3545:1000:720::1", - "network": "2a04:3545:1000:720::/64" - } - ], - "mac": "32:d5:ba:4a:63:e7", - "network_id": "03000000-0000-4000-8046-000000000000", - "type": "public" - }, + "address": "10.6.3.27", + "dhcp": true, + "dns": [], + "family": "IPv4", + "floating": false, + "gateway": "10.6.0.1", + "network": "10.6.0.0/22" + } + ], + "mac": "32:d5:ba:4a:84:cc", + "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1", + "type": "utility" + }, + { + "index": 3, + "ip_addresses": [ { - "index": 4, - "ip_addresses": [ - { - "address": "172.30.1.10", - "dhcp": true, - "dns": [], - "family": "IPv4", - "floating": false, - "gateway": "172.30.1.1", - "network": "172.30.1.0/24" - } + "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7", + "dhcp": true, + "dns": [ + "2a04:3540:53::1", + "2a04:3544:53::1" ], - "mac": "32:d5:ba:4a:8a:e1", - "network_id": "035a0a4a-77b4-4de5-820d-189fc8135714", - "type": "private" + "family": "IPv6", + "floating": false, + "gateway": "2a04:3545:1000:720::1", + "network": "2a04:3545:1000:720::/64" + } + ], + "mac": "32:d5:ba:4a:63:e7", + "network_id": "03000000-0000-4000-8046-000000000000", + "type": "public" + }, + { + "index": 4, + "ip_addresses": [ + { + "address": "172.30.1.10", + "dhcp": true, + "dns": [], + "family": "IPv4", + "floating": false, + "gateway": "172.30.1.1", + "network": "172.30.1.0/24" } ], - "dns": [ - "94.237.127.9", - "94.237.40.9" - ] + "mac": "32:d5:ba:4a:8a:e1", + "network_id": "035a0a4a-77b4-4de5-820d-189fc8135714", + "type": "private" } - """ - - nic_configs = {} - macs_to_interfaces = cloudnet.get_interfaces_by_mac() - LOG.debug("NIC mapping: %s", macs_to_interfaces) - for raw_iface in config.get("interfaces"): - LOG.debug("Considering %s", raw_iface) - - mac_address = raw_iface.get("mac") - if mac_address not in macs_to_interfaces: - raise RuntimeError( - "Did not find network interface on system " - "with mac '%s'. Cannot apply configuration: %s" - % (mac_address, raw_iface) - ) - - iface_type = raw_iface.get("type") - sysfs_name = macs_to_interfaces.get(mac_address) - - LOG.debug( - "Found %s interface '%s' with address '%s' (index %d)", - iface_type, - sysfs_name, - mac_address, - raw_iface.get("index"), - ) - - interface = {"match": {"macaddress": mac_address}} - - iface_dns = config.get("dns") - dhcp_configured = False - - for ip_addr in raw_iface.get("ip_addresses"): - if ip_addr.get("dhcp"): - if ip_addr.get("family") == "IPv6": - interface["dhcp6"] = True - interface["accept-ra"] = True - if "dns" in ip_addr: - iface_dns = ip_addr.get("dns") - else: - interface["dhcp4"] = True - - dhcp_configured = True - continue - - if "/" not in ip_addr.get("network"): - LOG.error( - "CIDR notation missing from network, not adding %s", - ip_addr.get("address"), - ) - continue - static_address = "{}/{}".format( - ip_addr.get("address"), ip_addr.get("network").split("/")[1] - ) - - if "addresses" not in interface: - interface["addresses"] = [] - interface["addresses"].append(static_address) - - if ip_addr.get("gateway"): - interface[ - "gateway{}".format(6 if "family" == "IPv6" else 4) - ] = ip_addr.get("gateway") - - if ip_addr.get("dns") and\ - (ip_addr.get("dns") != config.get("dns")): - interface["nameservers"] = ip_addr.get("dns") - - if not dhcp_configured: - interface["nameservers"] = {"addresses": iface_dns} - - LOG.debug( - "Generated interface config for %s: %s", - sysfs_name, interface - ) - nic_configs[sysfs_name] = interface - - return {"version": 2, "ethernets": nic_configs} - - -def convert_to_network_config_v1(config): - """ - Convert the UpCloud network metadata description into - Cloud-init's version 1 netconfig format. - - Example JSON can be seen above in v2 conversion function. + ], + "dns": [ + "94.237.127.9", + "94.237.40.9" + ] + } """ def _get_subnet_config(ip_addr, dns): @@ -276,10 +190,7 @@ def _get_subnet_config(ip_addr, dns): return {"version": 1, "config": nic_configs} -def convert_network_config(config, version=2): - if version == 2: - return convert_to_network_config_v2(config) - +def convert_network_config(config): return convert_to_network_config_v1(config) diff --git a/tests/unittests/test_datasource/test_upcloud.py b/tests/unittests/test_datasource/test_upcloud.py index 1963add0406..cec48b4b8c0 100644 --- a/tests/unittests/test_datasource/test_upcloud.py +++ b/tests/unittests/test_datasource/test_upcloud.py @@ -290,54 +290,6 @@ def test_network_configuration(self, m_get_by_mac, mock_readmd): dns.get('address')[1] ) - @mock.patch('cloudinit.sources.helpers.upcloud.read_metadata') - @mock.patch('cloudinit.net.get_interfaces_by_mac') - def test_v1_network_configuration(self, m_get_by_mac, mock_readmd): - mock_readmd.return_value = UC_METADATA.copy() - - raw_ifaces = UC_METADATA.get('network').get('interfaces') - self.assertEqual(4, len(raw_ifaces)) - - m_get_by_mac.return_value = { - raw_ifaces[0].get('mac'): 'eth0', - raw_ifaces[1].get('mac'): 'eth1', - raw_ifaces[2].get('mac'): 'eth2', - raw_ifaces[3].get('mac'): 'eth3', - } - - ds = self.get_ds() - ds.perform_dhcp_setup = False - - # Version 1 is the default, so we're overriding the config option. - ds.network_config_version = 2 - - ret = ds.get_data() - self.assertTrue(ret) - - self.assertTrue(mock_readmd.called) - - netcfg = ds.network_config - - self.assertEqual(2, netcfg.get('version')) - - ethernets = netcfg.get('ethernets') - self.assertIsInstance(ethernets, dict) - self.assertEqual(4, len(ethernets.keys())) - self.assertTrue('dhcp4' in ethernets.get('eth0')) - self.assertTrue(ethernets.get('eth0').get('dhcp4')) - - self.assertEqual( - raw_ifaces[2].get('mac'), - ethernets.get('eth2').get('match').get('macaddress') - ) - self.assertEqual(1, len(ethernets.get('eth0').get('addresses'))) - - self.assertTrue('dhcp4' not in ethernets.get('eth2')) - self.assertTrue('dhcp6' in ethernets.get('eth2')) - self.assertTrue(ethernets.get('eth2').get('dhcp6')) - self.assertTrue('accept-ra' in ethernets.get('eth2')) - self.assertTrue(ethernets.get('eth2').get('accept-ra')) - class TestUpCloudDatasourceLoading(CiTestCase): def test_get_datasource_list_returns_in_local(self): From 03e37b9b5601ee4080662e0b120c9573cbdcae2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Sat, 6 Feb 2021 15:37:51 +0200 Subject: [PATCH 15/16] UpCloud datasource: retries to 30 --- cloudinit/sources/DataSourceUpCloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py index 11b5857f2b8..209b9672a7a 100644 --- a/cloudinit/sources/DataSourceUpCloud.py +++ b/cloudinit/sources/DataSourceUpCloud.py @@ -20,7 +20,7 @@ # Wait for a up to a minute, retrying the meta-data server # every 2 seconds. -MD_RETRIES = 5 +MD_RETRIES = 30 MD_TIMEOUT = 2 MD_WAIT_RETRY = 2 From c605655ca5034655c4092742e2405a0d97ca520a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antti=20Myyr=C3=A4?= Date: Sat, 6 Feb 2021 16:00:22 +0200 Subject: [PATCH 16/16] UpCloud: update for docs --- doc/rtd/topics/datasources/upcloud.rst | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/doc/rtd/topics/datasources/upcloud.rst b/doc/rtd/topics/datasources/upcloud.rst index ecba7c2a067..0b7a9bb09b1 100644 --- a/doc/rtd/topics/datasources/upcloud.rst +++ b/doc/rtd/topics/datasources/upcloud.rst @@ -7,27 +7,18 @@ The `UpCloud`_ datasource consumes information from UpCloud's `metadata service`_. This metadata service serves information about the running server via HTTP over the address 169.254.169.254 available in every DHCP-configured interface. The metadata API endpoints are fully described in -UpCloud documentation at +UpCloud API documentation at `https://developers.upcloud.com/1.3/8-servers/#metadata-service `_. -Configuration -------------- +Providing user-data +------------------- -UpCloud's datasource can be configured as follows: - - datasource: - UpCloud: - retries: 5 - timeout: 2 - -- *retries*: Determines the number of times to attempt to connect to the - metadata service -- *timeout*: Determines the timeout in seconds to wait for a response from the - metadata service +When creating a server, user-data is provided by specifying it as `user_data` +in the API or via the server creation tool in the control panel. User-data is +immutable during server's lifetime and can be removed by deleting the server. .. _UpCloud: https://upcloud.com/ -.. _metadata service: https://developers.upcloud.com/1.3/8-servers/#metadata-service -.. _Full documentation: https://upcloud.com/community/tutorials/upcloud-metadata-service/ +.. _metadata service: https://upcloud.com/community/tutorials/upcloud-metadata-service/ .. vi: textwidth=78