From 49c00f86832db1b57560d89be4a9062f18d47e73 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Mon, 14 Feb 2022 14:13:58 +0000 Subject: [PATCH 01/32] Add CIX datasource with tests --- cloudinit/apport.py | 1 + cloudinit/settings.py | 1 + cloudinit/sources/DataSourceCloudCIX.py | 64 +++++++++++++++++++++ tests/unittests/sources/test_cloudcix.py | 71 ++++++++++++++++++++++++ tests/unittests/sources/test_common.py | 2 + tests/unittests/test_ds_identify.py | 8 +++ tools/ds-identify | 7 ++- 7 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 cloudinit/sources/DataSourceCloudCIX.py create mode 100644 tests/unittests/sources/test_cloudcix.py diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 92068aa9b49..3483977fc3f 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -23,6 +23,7 @@ "Azure", "Bigstep", "Brightbox", + "CloudCIX", "CloudSigma", "CloudStack", "DigitalOcean", diff --git a/cloudinit/settings.py b/cloudinit/settings.py index ecc1403bd9b..842061b43d4 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -33,6 +33,7 @@ "AliYun", "Vultr", "Ec2", + "CloudCIX", "CloudSigma", "CloudStack", "SmartOS", diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py new file mode 100644 index 00000000000..82fb52f2e63 --- /dev/null +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -0,0 +1,64 @@ +import json +import logging + +from cloudinit import dmi, sources, url_helper + +LOG = logging.getLogger(__name__) + + +class DataSourceCloudCIX(sources.DataSource): + + dsname = "CloudCIX" + base_url = "http://169.254.169.254" + + def _get_data(self): + """ + Datasources implement _get_data to setup metadata and userdata_raw. + + Minimally, the datasource should return a boolean True on success. + Subclasses of DataSource must implement _get_data which sets self.metadata, + vendordata_raw and userdata_raw. + """ + if not self.is_running_in_cloudcix(): + return False + + self.metadata = self.read_metadata() + self.userdata_raw = self.read_userdata() + + return True + + def is_running_in_cloudcix(self): + return dmi.read_dmi_data("system-product-name") == self.dsname + + def read_metadata(self): + metadata_url = url_helper.combine_url(self.base_url, "metadata") + return self.read_url(metadata_url) + + def read_userdata(self): + userdata_url = url_helper.combine_url(self.base_url, "userdata") + return self.read_url(userdata_url) + + def read_url(self, url): + response = url_helper.readurl( + url, + timeout=self.url_timeout, + sec_between=self.url_sec_between_retries, + retries=self.url_retries, + ) + if not response.ok(): + raise RuntimeError("unable to read metadata at %s" % url) + return json.loads(response.contents.decode()) + + +# Used to match classes to dependencies +datasources = [ + (DataSourceCloudCIX, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py new file mode 100644 index 00000000000..87061fbe5a3 --- /dev/null +++ b/tests/unittests/sources/test_cloudcix.py @@ -0,0 +1,71 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import distros, helpers, util +from cloudinit.sources.DataSourceCloudCIX import DataSourceCloudCIX +from tests.unittests.helpers import CiTestCase, mock + +METADATA = util.load_yaml( + """ + instance-id: 123456 + network-config: + config: + - name: eth0 + subnets: + - dns_nameservers: + - 213.133.99.99 + - 213.133.100.100 + - 213.133.98.98 + ipv4: true + type: dhcp + type: physical + version: 1 +""" +) + +USERDATA = b"""#cloud-config +runcmd: +- [ echo Hello, World >> /etc/greeting ] +""" + + +class TestDataSourceCloudCIX(CiTestCase): + """ + Test reading the meta-data + """ + + def setUp(self): + super(TestDataSourceCloudCIX, self).setUp() + self.paths = helpers.Paths({"run_dir": self.tmp_dir()}) + self.datasource = self._get_ds() + self.add_patch( + "cloudinit.dmi.read_dmi_data", + "m_read_dmi_data", + return_value="CloudCIX", + ) + + def _get_ds(self): + distro_cls = distros.fetch("ubuntu") + distro = distro_cls("ubuntu", cfg={}, paths=self.paths) + return DataSourceCloudCIX(sys_cfg={}, distro=distro, paths=self.paths) + + def test_identifying_cloudcix(self): + self.assertTrue(self.datasource.is_running_in_cloudcix()) + + self.m_read_dmi_data.return_value = "OnCloud9" + self.assertFalse(self.datasource.is_running_in_cloudcix()) + + @mock.patch( + "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_url" + ) + def test_reading_data(self, m_read_url): + def m_responses(url): + if url.endswith("metadata"): + return METADATA + elif url.endswith("userdata"): + return USERDATA + + m_read_url.side_effect = m_responses + + self.datasource.get_data() + self.assertEqual(self.datasource.metadata, METADATA) + self.assertEqual(self.datasource.userdata_raw, USERDATA) diff --git a/tests/unittests/sources/test_common.py b/tests/unittests/sources/test_common.py index a5bdb62905e..82b4035bf63 100644 --- a/tests/unittests/sources/test_common.py +++ b/tests/unittests/sources/test_common.py @@ -6,6 +6,7 @@ from cloudinit.sources import DataSourceAltCloud as AltCloud from cloudinit.sources import DataSourceAzure as Azure from cloudinit.sources import DataSourceBigstep as Bigstep +from cloudinit.sources import DataSourceCloudCIX as CloudCIX from cloudinit.sources import DataSourceCloudSigma as CloudSigma from cloudinit.sources import DataSourceCloudStack as CloudStack from cloudinit.sources import DataSourceConfigDrive as ConfigDrive @@ -58,6 +59,7 @@ AliYun.DataSourceAliYun, AltCloud.DataSourceAltCloud, Bigstep.DataSourceBigstep, + CloudCIX.DataSourceCloudCIX, CloudStack.DataSourceCloudStack, DSNone.DataSourceNone, Ec2.DataSourceEc2, diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 0b0de395b13..79f3f8d6189 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -726,6 +726,10 @@ def test_hetzner_found(self): """Hetzner cloud is identified in sys_vendor.""" self._test_ds_found("Hetzner") + def test_cloudcix_found(self): + """CloudCIX cloud is identified in dmi product-name""" + self._test_ds_found("CloudCIX") + def test_smartos_bhyve(self): """SmartOS cloud identified by SmartDC in dmi.""" self._test_ds_found("SmartOS-bhyve") @@ -987,6 +991,10 @@ def _print_run_output(rc, out, err, cfg, files): os.path.join(P_SEED_DIR, "azure", "ovf-env.xml"): "present\n", }, }, + "CloudCIX": { + "ds": "CloudCIX", + "files": {P_PRODUCT_NAME: "CloudCIX\n"}, + }, "Ec2-hvm": { "ds": "Ec2", "mocks": [{"name": "detect_virt", "RET": "kvm", "ret": 0}], diff --git a/tools/ds-identify b/tools/ds-identify index 794a96f48eb..699fd0ece8b 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -123,7 +123,7 @@ DS_MAYBE=2 DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. -DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ +DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep CloudCIX \ CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud VMware \ LXD" @@ -732,6 +732,11 @@ dscheck_CloudStack() { return $DS_NOT_FOUND } +dscheck_CloudCIX() { + dmi_product_name_matches "CloudCIX" && return $DS_FOUND + return $DS_NOT_FOUND +} + dscheck_Exoscale() { dmi_product_name_matches "Exoscale*" && return $DS_FOUND return $DS_NOT_FOUND From 3d1d13399f8431c4888d3e7e1995efe26f45966b Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Tue, 15 Feb 2022 11:02:15 +0000 Subject: [PATCH 02/32] Update sample metadata in tests --- tests/unittests/sources/test_cloudcix.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 87061fbe5a3..fa6e4954b4a 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -7,18 +7,10 @@ METADATA = util.load_yaml( """ instance-id: 123456 - network-config: - config: - - name: eth0 - subnets: - - dns_nameservers: - - 213.133.99.99 - - 213.133.100.100 - - 213.133.98.98 - ipv4: true - type: dhcp - type: physical - version: 1 + ip_addresses: + - private_ip: 10.0.0.2 + public_ip: 185.1.2.3 + subnet: 185.1.2.0/24 """ ) From 554419e11787d5cbffea3d36736c7d7be54ed2f2 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Wed, 23 Mar 2022 10:05:51 +0000 Subject: [PATCH 03/32] Don't try to read response json in helper function --- cloudinit/sources/DataSourceCloudCIX.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 82fb52f2e63..eabc8542ddf 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -32,11 +32,13 @@ def is_running_in_cloudcix(self): def read_metadata(self): metadata_url = url_helper.combine_url(self.base_url, "metadata") - return self.read_url(metadata_url) + response = self.read_url(metadata_url) + return json.loads(response.contents.decode()) def read_userdata(self): userdata_url = url_helper.combine_url(self.base_url, "userdata") - return self.read_url(userdata_url) + response = self.read_url(userdata_url) + return response.contents.decode() def read_url(self, url): response = url_helper.readurl( @@ -46,8 +48,8 @@ def read_url(self, url): retries=self.url_retries, ) if not response.ok(): - raise RuntimeError("unable to read metadata at %s" % url) - return json.loads(response.contents.decode()) + raise RuntimeError("Unable to read data at %s" % url) + return response # Used to match classes to dependencies From 1fd4423e0aceb7f990bb048f828a17a59a82032c Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Wed, 23 Mar 2022 15:53:25 +0000 Subject: [PATCH 04/32] Add test to check failing get_data call --- tests/unittests/sources/test_cloudcix.py | 47 ++++++++++++++---------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index fa6e4954b4a..9e128d08a72 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,18 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import distros, helpers, util +from cloudinit import distros, helpers from cloudinit.sources.DataSourceCloudCIX import DataSourceCloudCIX from tests.unittests.helpers import CiTestCase, mock -METADATA = util.load_yaml( - """ - instance-id: 123456 - ip_addresses: - - private_ip: 10.0.0.2 - public_ip: 185.1.2.3 - subnet: 185.1.2.0/24 -""" -) +METADATA = { + "instance_id": "12_34", + "ip_addresses": [ + { + "private_ip": "10.0.0.2", + "public_ip": "185.1.2.3", + "subnet": "10.0.0.1/24", + } + ], +} USERDATA = b"""#cloud-config runcmd: @@ -47,17 +48,25 @@ def test_identifying_cloudcix(self): self.assertFalse(self.datasource.is_running_in_cloudcix()) @mock.patch( - "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_url" + "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_metadata" ) - def test_reading_data(self, m_read_url): - def m_responses(url): - if url.endswith("metadata"): - return METADATA - elif url.endswith("userdata"): - return USERDATA - - m_read_url.side_effect = m_responses + @mock.patch( + "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_userdata" + ) + def test_reading_metadata_on_cloudcix( + self, m_read_userdata, m_read_metadata + ): + m_read_userdata.return_value = USERDATA + m_read_metadata.return_value = METADATA self.datasource.get_data() self.assertEqual(self.datasource.metadata, METADATA) self.assertEqual(self.datasource.userdata_raw, USERDATA) + + @mock.patch( + "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_metadata" + ) + def test_not_on_cloudcix_returns_false(self, m_read_metadata): + self.m_read_dmi_data.return_value = "WrongCloud" + self.assertFalse(self.datasource.get_data()) + m_read_metadata.assert_not_called() From 161ce9130c2c33cfe32d683d179a313a5e681965 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Wed, 23 Mar 2022 16:36:36 +0000 Subject: [PATCH 05/32] Update ca signers list --- tools/.github-cla-signers | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 4f443a883a1..93b86ab01a2 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -11,6 +11,7 @@ aswinrajamannar beezly bipinbachhao BirknerAlex +BrianKelleher bmhughes candlerb cawamata From 5f5b013df50da73c405ffefbdd02033c4c520d0f Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Thu, 24 Mar 2022 10:39:42 +0000 Subject: [PATCH 06/32] Remove un-used logger and comment --- cloudinit/sources/DataSourceCloudCIX.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index eabc8542ddf..9b1e1b1aab2 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -3,8 +3,6 @@ from cloudinit import dmi, sources, url_helper -LOG = logging.getLogger(__name__) - class DataSourceCloudCIX(sources.DataSource): @@ -12,19 +10,11 @@ class DataSourceCloudCIX(sources.DataSource): base_url = "http://169.254.169.254" def _get_data(self): - """ - Datasources implement _get_data to setup metadata and userdata_raw. - - Minimally, the datasource should return a boolean True on success. - Subclasses of DataSource must implement _get_data which sets self.metadata, - vendordata_raw and userdata_raw. - """ if not self.is_running_in_cloudcix(): return False self.metadata = self.read_metadata() self.userdata_raw = self.read_userdata() - return True def is_running_in_cloudcix(self): From 03c1a443257284ba02a3305747dd21b310b7b7f2 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Fri, 25 Mar 2022 11:11:52 +0000 Subject: [PATCH 07/32] Fix signature --- tools/.github-cla-signers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 93b86ab01a2..151b011cd71 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -11,7 +11,7 @@ aswinrajamannar beezly bipinbachhao BirknerAlex -BrianKelleher +BrinKe-dev bmhughes candlerb cawamata From 79877b9d42701d4998636c47cece7e76a51e446e Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Mon, 28 Mar 2022 09:36:44 +0100 Subject: [PATCH 08/32] Remove unused import --- cloudinit/sources/DataSourceCloudCIX.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 9b1e1b1aab2..d4466e885e5 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -1,5 +1,4 @@ import json -import logging from cloudinit import dmi, sources, url_helper From a02caf9321e890770efab4c125ba96d3a7d66251 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Wed, 30 Mar 2022 10:41:40 +0100 Subject: [PATCH 09/32] Add CloudCIX to the end of the DataSource list --- tools/ds-identify | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ds-identify b/tools/ds-identify index 699fd0ece8b..3c58614413b 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -123,10 +123,10 @@ DS_MAYBE=2 DI_DSNAME="" # this has to match the builtin list in cloud-init, it is what will # be searched if there is no setting found in config. -DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep CloudCIX \ +DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud VMware \ -LXD" +LXD CloudCIX" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" From b0ad6e161dfccaf8248c87161dd4ac8b8ed05d03 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Thu, 31 Mar 2022 17:02:48 +0100 Subject: [PATCH 10/32] Move reading metadata to separate function --- cloudinit/sources/DataSourceCloudCIX.py | 64 ++++++++++++++++--------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index d4466e885e5..4fb6960d66c 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -1,44 +1,62 @@ import json -from cloudinit import dmi, sources, url_helper +from cloudinit import log as logging +from cloudinit import dmi, sources, url_helper, util +LOG = logging.getLogger(__name__) class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" - base_url = "http://169.254.169.254" + base_url = "http://169.254.169.254/v1" def _get_data(self): if not self.is_running_in_cloudcix(): return False - self.metadata = self.read_metadata() - self.userdata_raw = self.read_userdata() + try: + md = read_metadata(self.base_url, self.get_url_params()) + except sources.InvalidMetaDataException as error: + LOG.debug(f"Failed to read data from CloudCIX datasource: {error}") + return False + + self.metadata = md['meta-data'] + self.userdata_raw = md['user-data'] return True def is_running_in_cloudcix(self): return dmi.read_dmi_data("system-product-name") == self.dsname - def read_metadata(self): - metadata_url = url_helper.combine_url(self.base_url, "metadata") - response = self.read_url(metadata_url) - return json.loads(response.contents.decode()) - - def read_userdata(self): - userdata_url = url_helper.combine_url(self.base_url, "userdata") - response = self.read_url(userdata_url) - return response.contents.decode() - - def read_url(self, url): - response = url_helper.readurl( - url, - timeout=self.url_timeout, - sec_between=self.url_sec_between_retries, - retries=self.url_retries, - ) + +def read_metadata(base_url, url_params): + md = {} + leaf_key_format_callback = ( + ("metadata", "meta-data", util.load_json), + ("userdata", "user-data", util.decode_binary), + ) + + for url_leaf, new_key, format_callback in leaf_key_format_callback: + try: + response = url_helper.readurl( + url=url_helper.combine_url(base_url, url_leaf), + retries=url_params.num_retries, + sec_between=url_params.sec_between_retries, + ) + except url_helper.UrlError as error: + raise sources.InvalidMetaDataException( + f"Failed to fetch IMDS {url_leaf}: {base_url}/{url_leaf}: {error}" + ) + if not response.ok(): - raise RuntimeError("Unable to read data at %s" % url) - return response + raise sources.InvalidMetaDataException( + f"No valid {url_leaf} found. URL {base_url}/{url_leaf} returned code {response.code}" + ) + + try: + md[new_key] = format_callback(response.contents) + except json.decoder.JSONDecodeError as exc: + raise sources.InvalidMetaDataException(f"Invalid JSON at {base_url}/{url_leaf}: {exc}") from exc + return md # Used to match classes to dependencies From 741326690699b0bf2f3a98682dee173fdb10e7a7 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Thu, 31 Mar 2022 17:21:15 +0100 Subject: [PATCH 11/32] Use util.log_time when fetching metadata --- cloudinit/sources/DataSourceCloudCIX.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 4fb6960d66c..db2ded87a9f 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -15,7 +15,15 @@ def _get_data(self): return False try: - md = read_metadata(self.base_url, self.get_url_params()) + md = util.log_time( + LOG.debug, + "Crawl of CloudCIX metadata service", + read_metadata, + kwargs={ + "base_url": self.base_url, + "url_params": self.get_url_params() + } + ) except sources.InvalidMetaDataException as error: LOG.debug(f"Failed to read data from CloudCIX datasource: {error}") return False From 2d2d951944a3bc789898fb41f9b7bed4a1ec7dd0 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Mon, 4 Apr 2022 15:20:29 +0100 Subject: [PATCH 12/32] Add CloudCIX ds docs. Allow config options --- cloudinit/sources/DataSourceCloudCIX.py | 30 +++++-- doc/rtd/topics/datasources/cloudcix.rst | 33 ++++++++ tests/unittests/sources/test_cloudcix.py | 101 ++++++++++++++++++----- 3 files changed, 138 insertions(+), 26 deletions(-) create mode 100644 doc/rtd/topics/datasources/cloudcix.rst diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index db2ded87a9f..6d4fa167c9f 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -1,15 +1,30 @@ import json +from cloudinit import dmi from cloudinit import log as logging -from cloudinit import dmi, sources, url_helper, util +from cloudinit import sources, url_helper, util LOG = logging.getLogger(__name__) +URL_TIMEOUT = 10 +URL_RETRIES = 5 +URL_SEC_BETWEEN_RETRIES = 1 + + class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" base_url = "http://169.254.169.254/v1" + def __init__(self, sys_cfg, distro, paths): + super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) + + self.url_timeout = self.ds_cfg.get("timeout", URL_TIMEOUT) + self.url_retries = self.ds_cfg.get("retries", URL_RETRIES) + self.wait_retry = self.ds_cfg.get( + "wait_retry", URL_SEC_BETWEEN_RETRIES + ) + def _get_data(self): if not self.is_running_in_cloudcix(): return False @@ -21,15 +36,15 @@ def _get_data(self): read_metadata, kwargs={ "base_url": self.base_url, - "url_params": self.get_url_params() - } + "url_params": self.get_url_params(), + }, ) except sources.InvalidMetaDataException as error: LOG.debug(f"Failed to read data from CloudCIX datasource: {error}") return False - self.metadata = md['meta-data'] - self.userdata_raw = md['user-data'] + self.metadata = md["meta-data"] + self.userdata_raw = md["user-data"] return True def is_running_in_cloudcix(self): @@ -49,6 +64,7 @@ def read_metadata(base_url, url_params): url=url_helper.combine_url(base_url, url_leaf), retries=url_params.num_retries, sec_between=url_params.sec_between_retries, + timeout=url_params.timeout_seconds, ) except url_helper.UrlError as error: raise sources.InvalidMetaDataException( @@ -63,7 +79,9 @@ def read_metadata(base_url, url_params): try: md[new_key] = format_callback(response.contents) except json.decoder.JSONDecodeError as exc: - raise sources.InvalidMetaDataException(f"Invalid JSON at {base_url}/{url_leaf}: {exc}") from exc + raise sources.InvalidMetaDataException( + f"Invalid JSON at {base_url}/{url_leaf}: {exc}" + ) from exc return md diff --git a/doc/rtd/topics/datasources/cloudcix.rst b/doc/rtd/topics/datasources/cloudcix.rst new file mode 100644 index 00000000000..daddd3725cb --- /dev/null +++ b/doc/rtd/topics/datasources/cloudcix.rst @@ -0,0 +1,33 @@ +.. _datasource_vultr: + +CloudCIX +======== + +CloudCIX serves metadata through an internal server, accessible at +``http://169.254.169.254/v1``. The metadata and userdata can be fetched at +the ``/metadata`` and ``/userdata`` paths respectively. + +CloudCIX instances are identified by the dmi Product Name. + +Configuration +------------- + +CloudCIX datasource has the following config options: + +:: + + datasource: + CloudCIX: + retries: 3 + timeout: 2 + wait_retry: 2 + + +- *retries*: The number of times the datasource should try to connect to the + metadata service +- *timeout*: How long in seconds to wait for a response from the metadata + service +- *wait*: How long in seconds to wait between consecutive requests to the + metadata service + +.. vi: textwidth=78 diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 9e128d08a72..ecfeb786bd2 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,7 +1,8 @@ # This file is part of cloud-init. See LICENSE file for license information. +import json -from cloudinit import distros, helpers -from cloudinit.sources.DataSourceCloudCIX import DataSourceCloudCIX +from cloudinit import distros, helpers, sources, url_helper +from cloudinit.sources import DataSourceCloudCIX as ds_mod from tests.unittests.helpers import CiTestCase, mock METADATA = { @@ -39,7 +40,9 @@ def setUp(self): def _get_ds(self): distro_cls = distros.fetch("ubuntu") distro = distro_cls("ubuntu", cfg={}, paths=self.paths) - return DataSourceCloudCIX(sys_cfg={}, distro=distro, paths=self.paths) + return ds_mod.DataSourceCloudCIX( + sys_cfg={}, distro=distro, paths=self.paths + ) def test_identifying_cloudcix(self): self.assertTrue(self.datasource.is_running_in_cloudcix()) @@ -47,25 +50,83 @@ def test_identifying_cloudcix(self): self.m_read_dmi_data.return_value = "OnCloud9" self.assertFalse(self.datasource.is_running_in_cloudcix()) - @mock.patch( - "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_metadata" - ) - @mock.patch( - "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_userdata" - ) - def test_reading_metadata_on_cloudcix( - self, m_read_userdata, m_read_metadata - ): - m_read_userdata.return_value = USERDATA - m_read_metadata.return_value = METADATA - - self.datasource.get_data() + @mock.patch("cloudinit.url_helper.readurl") + def test_reading_metadata_on_cloudcix(self, m_readurl): + def url_responses(url, **params): + if url.endswith("metadata"): + return url_helper.StringResponse(json.dumps(METADATA)) + elif url.endswith("userdata"): + return url_helper.StringResponse(USERDATA) + return None + + m_readurl.side_effect = url_responses + + self.assertTrue(self.datasource.get_data()) self.assertEqual(self.datasource.metadata, METADATA) - self.assertEqual(self.datasource.userdata_raw, USERDATA) + self.assertEqual(self.datasource.userdata_raw, USERDATA.decode()) + + def test_setting_config_options(self): + cix_options = { + "timeout": 1234, + "retries": 5678, + } + sys_cfg = { + "datasource": { + "CloudCIX": cix_options, + } + } + + # Instantiate a new datasource + distro_cls = distros.fetch("ubuntu") + distro = distro_cls("ubuntu", cfg={}, paths=self.paths) + new_ds = ds_mod.DataSourceCloudCIX( + sys_cfg=sys_cfg, distro=distro, paths=self.paths + ) + self.assertEqual(new_ds.url_timeout, cix_options["timeout"]) + self.assertEqual(new_ds.url_retries, cix_options["retries"]) + + @mock.patch("cloudinit.url_helper.readurl") + def test_read_metadata_cannot_contact_imds(self, m_readurl): + def url_responses(url, **params): + raise url_helper.UrlError("No route") + + m_readurl.side_effect = url_responses + + self.assertFalse(self.datasource.get_data()) + self.assertFalse(getattr(self.datasource, "metadata")) + self.assertFalse(getattr(self.datasource, "userdata_raw")) + + with self.assertRaises(sources.InvalidMetaDataException): + ds_mod.read_metadata( + self.datasource.base_url, self.datasource.get_url_params() + ) + + @mock.patch("cloudinit.url_helper.readurl") + def test_read_metadata_gets_bad_response(self, m_readurl): + def url_responses(url, **params): + return url_helper.StringResponse("", code=403) + + m_readurl.side_effect = url_responses + + with self.assertRaises(sources.InvalidMetaDataException): + ds_mod.read_metadata( + self.datasource.base_url, self.datasource.get_url_params() + ) + + @mock.patch("cloudinit.url_helper.readurl") + def test_read_metadata_gets_malformed_response(self, m_readurl): + def url_responses(url, **params): + bad_json = json.dumps(METADATA)[:-2] + return url_helper.StringResponse(bad_json, code=200) + + m_readurl.side_effect = url_responses + + with self.assertRaises(sources.InvalidMetaDataException): + ds_mod.read_metadata( + self.datasource.base_url, self.datasource.get_url_params() + ) - @mock.patch( - "cloudinit.sources.DataSourceCloudCIX.DataSourceCloudCIX.read_metadata" - ) + @mock.patch("cloudinit.sources.DataSourceCloudCIX.read_metadata") def test_not_on_cloudcix_returns_false(self, m_read_metadata): self.m_read_dmi_data.return_value = "WrongCloud" self.assertFalse(self.datasource.get_data()) From 7ae04ffabbe0f5498a2aac15a9d5bd10bc70406e Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Mon, 4 Apr 2022 15:38:01 +0100 Subject: [PATCH 13/32] Use lazy evaluation for logging message --- cloudinit/sources/DataSourceCloudCIX.py | 4 +++- doc/rtd/topics/datasources/cloudcix.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 6d4fa167c9f..61f8d805197 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -40,7 +40,9 @@ def _get_data(self): }, ) except sources.InvalidMetaDataException as error: - LOG.debug(f"Failed to read data from CloudCIX datasource: {error}") + LOG.debug( + "Failed to read data from CloudCIX datasource: %error", error + ) return False self.metadata = md["meta-data"] diff --git a/doc/rtd/topics/datasources/cloudcix.rst b/doc/rtd/topics/datasources/cloudcix.rst index daddd3725cb..bba2c22baa2 100644 --- a/doc/rtd/topics/datasources/cloudcix.rst +++ b/doc/rtd/topics/datasources/cloudcix.rst @@ -1,4 +1,4 @@ -.. _datasource_vultr: +.. _datasource_cloudcix: CloudCIX ======== From 49a89cb091afae1e6eb913acb8753664841c884a Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Mon, 4 Apr 2022 17:16:30 +0100 Subject: [PATCH 14/32] Fix logging message --- cloudinit/sources/DataSourceCloudCIX.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 61f8d805197..5326306b573 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -41,7 +41,7 @@ def _get_data(self): ) except sources.InvalidMetaDataException as error: LOG.debug( - "Failed to read data from CloudCIX datasource: %error", error + "Failed to read data from CloudCIX datasource: %s", error ) return False From dcd1c212e462afee0acf1b43d4efb27fe9fa89e9 Mon Sep 17 00:00:00 2001 From: BrinKe-dev Date: Wed, 6 Apr 2022 13:58:30 +0100 Subject: [PATCH 15/32] Generate v1 metadata url --- cloudinit/sources/DataSourceCloudCIX.py | 10 +++++++--- cloudinit/url_helper.py | 4 +--- tests/unittests/sources/test_cloudcix.py | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 5326306b573..76f9fec68f8 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -6,6 +6,8 @@ LOG = logging.getLogger(__name__) +DS_BASE_URL = "http://169.254.169.254/" + URL_TIMEOUT = 10 URL_RETRIES = 5 URL_SEC_BETWEEN_RETRIES = 1 @@ -14,11 +16,11 @@ class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" - base_url = "http://169.254.169.254/v1" def __init__(self, sys_cfg, distro, paths): super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) + self.base_url = url_helper.combine_url(DS_BASE_URL, "v1") self.url_timeout = self.ds_cfg.get("timeout", URL_TIMEOUT) self.url_retries = self.ds_cfg.get("retries", URL_RETRIES) self.wait_retry = self.ds_cfg.get( @@ -70,12 +72,14 @@ def read_metadata(base_url, url_params): ) except url_helper.UrlError as error: raise sources.InvalidMetaDataException( - f"Failed to fetch IMDS {url_leaf}: {base_url}/{url_leaf}: {error}" + f"Failed to fetch IMDS {url_leaf}: " + f"{base_url}/{url_leaf}: {error}" ) if not response.ok(): raise sources.InvalidMetaDataException( - f"No valid {url_leaf} found. URL {base_url}/{url_leaf} returned code {response.code}" + f"No valid {url_leaf} found. " + f"URL {base_url}/{url_leaf} returned code {response.code}" ) try: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 790e2fbfad1..4acd0fca681 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -95,9 +95,7 @@ def __init__(self, contents, code=200): self.url = None def ok(self, *args, **kwargs): - if self.code != 200: - return False - return True + return self.code == 200 def __str__(self): return self.contents.decode("utf-8") diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index ecfeb786bd2..5099ff9e135 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -36,6 +36,7 @@ def setUp(self): "m_read_dmi_data", return_value="CloudCIX", ) + self.base_url = self.datasource.base_url def _get_ds(self): distro_cls = distros.fetch("ubuntu") @@ -98,7 +99,7 @@ def url_responses(url, **params): with self.assertRaises(sources.InvalidMetaDataException): ds_mod.read_metadata( - self.datasource.base_url, self.datasource.get_url_params() + self.base_url, self.datasource.get_url_params() ) @mock.patch("cloudinit.url_helper.readurl") @@ -110,7 +111,7 @@ def url_responses(url, **params): with self.assertRaises(sources.InvalidMetaDataException): ds_mod.read_metadata( - self.datasource.base_url, self.datasource.get_url_params() + self.base_url, self.datasource.get_url_params() ) @mock.patch("cloudinit.url_helper.readurl") @@ -123,7 +124,7 @@ def url_responses(url, **params): with self.assertRaises(sources.InvalidMetaDataException): ds_mod.read_metadata( - self.datasource.base_url, self.datasource.get_url_params() + self.base_url, self.datasource.get_url_params() ) @mock.patch("cloudinit.sources.DataSourceCloudCIX.read_metadata") From bd850a75f75fae4fdd97f50d683bbf6173737305 Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 19 Mar 2024 10:23:51 +0000 Subject: [PATCH 16/32] Make cloudcix ds use DHCP --- cloudinit/sources/DataSourceCloudCIX.py | 222 ++++++++++++++++-------- 1 file changed, 152 insertions(+), 70 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 76f9fec68f8..d9242673e06 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -1,105 +1,187 @@ -import json +# This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import dmi -from cloudinit import log as logging -from cloudinit import sources, url_helper, util +import base64 +import json +import logging +from typing import Optional +from requests.exceptions import ConnectionError +from cloudinit import atomic_helper, dmi, helpers +from cloudinit import net, sources, url_helper, util +from cloudinit.sources.helpers import ec2 +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralDHCPv4, EphemeralIPNetwork LOG = logging.getLogger(__name__) -DS_BASE_URL = "http://169.254.169.254/" +METADATA_URLS = ["http://169.254.169.254"] + +METADATA_VERSION = 2 -URL_TIMEOUT = 10 -URL_RETRIES = 5 -URL_SEC_BETWEEN_RETRIES = 1 +URL_MAX_WAIT = 5 +URL_TIMEOUT = 5 +URL_RETRIES = 6 class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" + _metadata_url: str + def __init__(self, sys_cfg, distro, paths): super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) - - self.base_url = url_helper.combine_url(DS_BASE_URL, "v1") - self.url_timeout = self.ds_cfg.get("timeout", URL_TIMEOUT) - self.url_retries = self.ds_cfg.get("retries", URL_RETRIES) - self.wait_retry = self.ds_cfg.get( - "wait_retry", URL_SEC_BETWEEN_RETRIES - ) + LOG.debug("Initializing the CIX datasource") + self._metadata_url = None + self.max_wait = URL_MAX_WAIT + self.url_timeout = URL_TIMEOUT def _get_data(self): - if not self.is_running_in_cloudcix(): - return False + """Fetch the user data, the metadata and the VM password + from the metadata service. + Please refer to the datasource documentation for details on how the + metadata server and password server are crawled. + """ try: - md = util.log_time( - LOG.debug, - "Crawl of CloudCIX metadata service", - read_metadata, - kwargs={ - "base_url": self.base_url, - "url_params": self.get_url_params(), - }, - ) - except sources.InvalidMetaDataException as error: - LOG.debug( - "Failed to read data from CloudCIX datasource: %s", error - ) + with EphemeralIPNetwork( + self.distro, interface=net.find_fallback_nic(), ipv6=True, ipv4=True + ): + crawled_data = util.log_time( + logfunc=LOG.debug, + msg=f"Crawl of metadata service", + func=self.crawl_metadata_service, + ) + except NoDHCPLeaseError as e: + LOG.error("Bailing, DHCP exception: %s", e) + return False + + if not crawled_data: return False - self.metadata = md["meta-data"] - self.userdata_raw = md["user-data"] + self.metadata = crawled_data["metadata"] + self.userdata_raw = crawled_data["userdata_raw"] + return True - def is_running_in_cloudcix(self): - return dmi.read_dmi_data("system-product-name") == self.dsname + def crawl_metadata_service(self) -> dict: + # Wait for metadata server + md_url = self.determine_md_url() + if md_url is None: + return {} + data = {} + data["metadata"] = read_metadata(md_url) + if data["metadata"] is None: + return {} -def read_metadata(base_url, url_params): - md = {} - leaf_key_format_callback = ( - ("metadata", "meta-data", util.load_json), - ("userdata", "user-data", util.decode_binary), - ) + data["userdata_raw"] = read_userdata_raw(md_url) + if data["userdata_raw"] is None: + return {} - for url_leaf, new_key, format_callback in leaf_key_format_callback: - try: - response = url_helper.readurl( - url=url_helper.combine_url(base_url, url_leaf), - retries=url_params.num_retries, - sec_between=url_params.sec_between_retries, - timeout=url_params.timeout_seconds, - ) - except url_helper.UrlError as error: - raise sources.InvalidMetaDataException( - f"Failed to fetch IMDS {url_leaf}: " - f"{base_url}/{url_leaf}: {error}" - ) - - if not response.ok(): - raise sources.InvalidMetaDataException( - f"No valid {url_leaf} found. " - f"URL {base_url}/{url_leaf} returned code {response.code}" - ) + return data + + def determine_md_url(self) -> Optional[str]: + if self._metadata_url: + return self._metadata_url + + # Try to reach the metadata server + base_url, _ = url_helper.wait_for_url( + METADATA_URLS, + max_wait=self.max_wait, + timeout=self.url_timeout, + ) + if not base_url: + return None + + # Find the highest supported metadata version + md_url = None + for version in range(METADATA_VERSION, 0, -1): + url = url_helper.combine_url(base_url, "v{0}".format(version), "metadata") + try: + response = url_helper.readurl(url, timeout=self.url_timeout) + except url_helper.UrlError as e: + LOG.debug("URL %s raised exception %s", url, e) + continue + + if response.ok(): + md_url = url_helper.combine_url(base_url, "v{0}".format(version)) + break + else: + LOG.debug("No metadata found at URL %s", url) + + self._metadata_url = md_url + return self._metadata_url + + @staticmethod + def ds_detect(): + product_name = dmi.read_dmi_data("system-product") + if product_name == self.dsname: + return True + + return False + + @property + def network_config(self): + if self._net_cfg: + return self._net_cfg + + if not self.metadata: + return None + self._net_cfg = self._generate_net_cfg(self.metadata) + return self._net_cfg + + def _generate_net_cfg(self, metadata): + netcfg = {"version": 2, "ethernets": {}} + macs_to_nics = net.get_interfaces_by_mac() + + interface_map = {} + for iface in metadata["network"]["interfaces"]: + name = macs_to_nics.get(iface["mac_address"]) + if name is None: + LOG.warning("Metadata mac address %s not found.", iface["mac_address"]) + continue + interfaces["name"] = iface + + for name, iface in interfaces.items(): + netcfg["ethernets"][name] = { + "set-name": name, + "match": { + "macaddress": iface["mac_address"].lower(), + }, + "addresses": iface["addresses"] + } + + return netcfg - try: - md[new_key] = format_callback(response.contents) - except json.decoder.JSONDecodeError as exc: - raise sources.InvalidMetaDataException( - f"Invalid JSON at {base_url}/{url_leaf}: {exc}" - ) from exc - return md + +def read_metadata(md_url) -> Optional[str]: + url = url_helper.combine_url(md_url, "metadata") + try: + response = url_helper.readurl(url) + metadata = json.loads(response.contents.decode()) + except (url_helper.UrlError, json.JSONDecodeError) as e: + LOG.warning("Failed to read metadata. Cause: %s", e) + metadata = None + return metadata + + +def read_userdata_raw(md_url) -> Optional[str]: + url = url_helper.combine_url(md_url, "userdata") + try: + response = url_helper.readurl(url) + userdata_raw = util.maybe_b64decode(response.contents) + except url_helper.UrlError as e: + LOG.warning("Failed to read userdata. Cause: %s", e) + userdata_raw = None + return userdata_raw # Used to match classes to dependencies datasources = [ - (DataSourceCloudCIX, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceCloudCIX, (sources.DEP_FILESYSTEM,)), ] # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) - - -# vi: ts=4 expandtab From 4afc9629f254795220d910c9600f7f2960e1a2bd Mon Sep 17 00:00:00 2001 From: Brian Date: Thu, 21 Mar 2024 09:46:20 +0000 Subject: [PATCH 17/32] Add tests and ds_config options --- cloudinit/sources/DataSourceCloudCIX.py | 135 ++++++++++------- tests/unittests/sources/test_cloudcix.py | 183 +++++++++++++++-------- 2 files changed, 205 insertions(+), 113 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index d9242673e06..6b281457d9c 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -14,12 +14,16 @@ LOG = logging.getLogger(__name__) METADATA_URLS = ["http://169.254.169.254"] +METADATA_VERSION = 1 -METADATA_VERSION = 2 +BUILTIN_DS_CONFIG = { + "retries": 3, + "timeout": 5, + "wait": 5, +} -URL_MAX_WAIT = 5 -URL_TIMEOUT = 5 -URL_RETRIES = 6 + +CLOUDCIX_DMI_NAME = "CloudCIX" class DataSourceCloudCIX(sources.DataSource): @@ -30,10 +34,18 @@ class DataSourceCloudCIX(sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) - LOG.debug("Initializing the CIX datasource") + self.distro = distro + self.ds_cfg = util.mergemanydict( + [ + util.get_cfg_by_path(sys_cfg, ["datasource", "CloudCIX"], {}), + BUILTIN_DS_CONFIG, + ] + ) self._metadata_url = None - self.max_wait = URL_MAX_WAIT - self.url_timeout = URL_TIMEOUT + self._net_cfg = None + self.url_retries = self.ds_cfg.get("retries") + self.url_timeout = self.ds_cfg.get("timeout") + self.url_sec_between_retries = self.ds_cfg.get("wait") def _get_data(self): """Fetch the user data, the metadata and the VM password @@ -44,7 +56,10 @@ def _get_data(self): """ try: with EphemeralIPNetwork( - self.distro, interface=net.find_fallback_nic(), ipv6=True, ipv4=True + self.distro, + interface=net.find_fallback_nic(), + ipv6=True, + ipv4=True, ): crawled_data = util.log_time( logfunc=LOG.debug, @@ -54,30 +69,25 @@ def _get_data(self): except NoDHCPLeaseError as e: LOG.error("Bailing, DHCP exception: %s", e) return False - - if not crawled_data: + except sources.InvalidMetaDataException as error: + LOG.debug( + "Failed to read data from CloudCIX datasource: %s", error + ) return False - self.metadata = crawled_data["metadata"] - self.userdata_raw = crawled_data["userdata_raw"] + self.metadata = crawled_data["meta-data"] + self.userdata_raw = util.decode_binary(crawled_data["user-data"]) return True def crawl_metadata_service(self) -> dict: - # Wait for metadata server md_url = self.determine_md_url() if md_url is None: - return {} - - data = {} - data["metadata"] = read_metadata(md_url) - if data["metadata"] is None: - return {} - - data["userdata_raw"] = read_userdata_raw(md_url) - if data["userdata_raw"] is None: - return {} + raise sources.InvalidMetaDataException( + f"Could not reach determine MetaData url" + ) + data = read_metadata(md_url, self.get_url_params()) return data def determine_md_url(self) -> Optional[str]: @@ -85,10 +95,11 @@ def determine_md_url(self) -> Optional[str]: return self._metadata_url # Try to reach the metadata server + url_params = self.get_url_params() base_url, _ = url_helper.wait_for_url( METADATA_URLS, - max_wait=self.max_wait, - timeout=self.url_timeout, + max_wait=url_params.max_wait_seconds, + timeout=url_params.timeout_seconds, ) if not base_url: return None @@ -96,7 +107,9 @@ def determine_md_url(self) -> Optional[str]: # Find the highest supported metadata version md_url = None for version in range(METADATA_VERSION, 0, -1): - url = url_helper.combine_url(base_url, "v{0}".format(version), "metadata") + url = url_helper.combine_url( + base_url, "v{0}".format(version), "metadata" + ) try: response = url_helper.readurl(url, timeout=self.url_timeout) except url_helper.UrlError as e: @@ -104,7 +117,9 @@ def determine_md_url(self) -> Optional[str]: continue if response.ok(): - md_url = url_helper.combine_url(base_url, "v{0}".format(version)) + md_url = url_helper.combine_url( + base_url, "v{0}".format(version) + ) break else: LOG.debug("No metadata found at URL %s", url) @@ -115,10 +130,7 @@ def determine_md_url(self) -> Optional[str]: @staticmethod def ds_detect(): product_name = dmi.read_dmi_data("system-product") - if product_name == self.dsname: - return True - - return False + return product_name == CLOUDCIX_DMI_NAME @property def network_config(self): @@ -134,11 +146,12 @@ def _generate_net_cfg(self, metadata): netcfg = {"version": 2, "ethernets": {}} macs_to_nics = net.get_interfaces_by_mac() - interface_map = {} for iface in metadata["network"]["interfaces"]: name = macs_to_nics.get(iface["mac_address"]) if name is None: - LOG.warning("Metadata mac address %s not found.", iface["mac_address"]) + LOG.warning( + "Metadata mac address %s not found.", iface["mac_address"] + ) continue interfaces["name"] = iface @@ -148,32 +161,46 @@ def _generate_net_cfg(self, metadata): "match": { "macaddress": iface["mac_address"].lower(), }, - "addresses": iface["addresses"] + "addresses": iface["addresses"], } return netcfg -def read_metadata(md_url) -> Optional[str]: - url = url_helper.combine_url(md_url, "metadata") - try: - response = url_helper.readurl(url) - metadata = json.loads(response.contents.decode()) - except (url_helper.UrlError, json.JSONDecodeError) as e: - LOG.warning("Failed to read metadata. Cause: %s", e) - metadata = None - return metadata - - -def read_userdata_raw(md_url) -> Optional[str]: - url = url_helper.combine_url(md_url, "userdata") - try: - response = url_helper.readurl(url) - userdata_raw = util.maybe_b64decode(response.contents) - except url_helper.UrlError as e: - LOG.warning("Failed to read userdata. Cause: %s", e) - userdata_raw = None - return userdata_raw +def read_metadata(base_url, url_params): + md = {} + leaf_key_format_callback = ( + ("metadata", "meta-data", util.load_json), + ("userdata", "user-data", util.maybe_64decode), + ) + + for url_leaf, new_key, format_callback in leaf_key_format_callback: + try: + response = url_helper.readurl( + url=url_helper.combine_url(base_url, url_leaf), + retries=url_params.num_retries, + sec_between=url_params.sec_between_retries, + timeout=url_params.timeout_seconds, + ) + except url_helper.UrlError as error: + raise sources.InvalidMetaDataException( + f"Failed to fetch IMDS {url_leaf}: " + f"{base_url}/{url_leaf}: {error}" + ) + + if not response.ok(): + raise sources.InvalidMetaDataException( + f"No valid {url_leaf} found. " + f"URL {base_url}/{url_leaf} returned code {response.code}" + ) + + try: + md[new_key] = format_callback(response.contents) + except json.decoder.JSONDecodeError as exc: + raise sources.InvalidMetaDataException( + f"Invalid JSON at {base_url}/{url_leaf}: {exc}" + ) from exc + return md # Used to match classes to dependencies diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 5099ff9e135..8340d6648b2 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,19 +1,32 @@ # This file is part of cloud-init. See LICENSE file for license information. import json -from cloudinit import distros, helpers, sources, url_helper -from cloudinit.sources import DataSourceCloudCIX as ds_mod +from cloudinit import distros, helpers, sources, url_helper as uh +from cloudinit.sources import ( + DataSourceCloudCIX as ds_mod, + InvalidMetaDataException, +) from tests.unittests.helpers import CiTestCase, mock METADATA = { "instance_id": "12_34", - "ip_addresses": [ - { - "private_ip": "10.0.0.2", - "public_ip": "185.1.2.3", - "subnet": "10.0.0.1/24", - } - ], + "network": { + "interfaces": [ + { + "mac_address": "ab:cd:ef:00:01:02", + "addresses": [ + "10.0.0.2/24", + "192.168.0.2/24", + ], + }, + { + "mac_address": "12:34:56:ab:cd:ef", + "addresses": [ + "10.10.10.2/24", + ], + }, + ] + }, } USERDATA = b"""#cloud-config @@ -22,6 +35,28 @@ """ +def mock_imds(base_url, version): + # Create a function that mocks responses from a metadata server + version_str = f"v{version}" + + def func(url, **urlparams): + + if url == base_url: + return uh.StringResponse("Metadata enabled") + + md_endpoint = uh.combine_url(base_url, version_str, "metadata") + if url == md_endpoint: + return uh.StringResponse(json.dumps(METADATA).encode()) + + userdata_endpoint = uh.combine_url(base_url, version_str, "userdata") + if url == userdata_endpoint: + return uh.StringResponse(USERDATA) + + return uh.StringResponse("Not Found", code=404) + + return func + + class TestDataSourceCloudCIX(CiTestCase): """ Test reading the meta-data @@ -31,40 +66,35 @@ def setUp(self): super(TestDataSourceCloudCIX, self).setUp() self.paths = helpers.Paths({"run_dir": self.tmp_dir()}) self.datasource = self._get_ds() + self.allowed_subp = True self.add_patch( "cloudinit.dmi.read_dmi_data", "m_read_dmi_data", return_value="CloudCIX", ) - self.base_url = self.datasource.base_url def _get_ds(self): distro_cls = distros.fetch("ubuntu") distro = distro_cls("ubuntu", cfg={}, paths=self.paths) return ds_mod.DataSourceCloudCIX( - sys_cfg={}, distro=distro, paths=self.paths + sys_cfg={ + "datasource": { + "CloudCIX": { + "retries": 1, + "timeout": 0, + "wait": 0, + }, + } + }, + distro=distro, + paths=self.paths, ) def test_identifying_cloudcix(self): - self.assertTrue(self.datasource.is_running_in_cloudcix()) + self.assertTrue(self.datasource.ds_detect()) self.m_read_dmi_data.return_value = "OnCloud9" - self.assertFalse(self.datasource.is_running_in_cloudcix()) - - @mock.patch("cloudinit.url_helper.readurl") - def test_reading_metadata_on_cloudcix(self, m_readurl): - def url_responses(url, **params): - if url.endswith("metadata"): - return url_helper.StringResponse(json.dumps(METADATA)) - elif url.endswith("userdata"): - return url_helper.StringResponse(USERDATA) - return None - - m_readurl.side_effect = url_responses - - self.assertTrue(self.datasource.get_data()) - self.assertEqual(self.datasource.metadata, METADATA) - self.assertEqual(self.datasource.userdata_raw, USERDATA.decode()) + self.assertFalse(self.datasource.ds_detect()) def test_setting_config_options(self): cix_options = { @@ -87,48 +117,83 @@ def test_setting_config_options(self): self.assertEqual(new_ds.url_retries, cix_options["retries"]) @mock.patch("cloudinit.url_helper.readurl") - def test_read_metadata_cannot_contact_imds(self, m_readurl): - def url_responses(url, **params): - raise url_helper.UrlError("No route") + def test_determine_md_url(self, m_readurl): + base_url = ds_mod.METADATA_URLS[0] + version = ds_mod.METADATA_VERSION + m_readurl.side_effect = mock_imds(base_url, version) - m_readurl.side_effect = url_responses + md_url = self._get_ds().determine_md_url() - self.assertFalse(self.datasource.get_data()) - self.assertFalse(getattr(self.datasource, "metadata")) - self.assertFalse(getattr(self.datasource, "userdata_raw")) + expected_url = uh.combine_url( + ds_mod.METADATA_URLS[0], + f"v{version}", + ) + self.assertEqual(md_url, expected_url) - with self.assertRaises(sources.InvalidMetaDataException): - ds_mod.read_metadata( - self.base_url, self.datasource.get_url_params() - ) + @mock.patch("cloudinit.url_helper.readurl") + def test_reading_metadata_on_cloudcix(self, m_readurl): + base_url = ds_mod.METADATA_URLS[0] + version = ds_mod.METADATA_VERSION + m_readurl.side_effect = mock_imds(base_url, version) + + self.assertTrue(self.datasource.get_data()) + self.assertEqual(self.datasource.metadata, METADATA) + self.assertEqual(self.datasource.userdata_raw, USERDATA.decode()) @mock.patch("cloudinit.url_helper.readurl") - def test_read_metadata_gets_bad_response(self, m_readurl): - def url_responses(url, **params): - return url_helper.StringResponse("", code=403) + def test_failing_imds_endpoints(self, m_readurl): + # Set up an empty imds + endpoints = dict() - m_readurl.side_effect = url_responses + def faulty_imds(url, **urlparams): + if url in endpoints: + return endpoints[url] + return uh.StringResponse(b"Not Found", code=404) - with self.assertRaises(sources.InvalidMetaDataException): - ds_mod.read_metadata( - self.base_url, self.datasource.get_url_params() - ) + m_readurl.side_effect = faulty_imds + + self.assertRaises( + sources.InvalidMetaDataException, + self.datasource.crawl_metadata_service, + ) + + # Make imds respond to healthcheck + base_url = ds_mod.METADATA_URLS[0] + endpoints[base_url] = uh.StringResponse("Metadata enabled") + + self.assertRaises( + sources.InvalidMetaDataException, + self.datasource.crawl_metadata_service, + ) + + # Make imds serve metadata + version_str = f"v{ds_mod.METADATA_VERSION}" + md_url = uh.combine_url(base_url, version_str, "metadata") + endpoints[md_url] = uh.StringResponse(json.dumps(METADATA).encode()) + + self.assertRaises( + sources.InvalidMetaDataException, + self.datasource.crawl_metadata_service, + ) + + # Make imds serve userdata + ud_url = uh.combine_url(base_url, version_str, "userdata") + endpoints[ud_url] = uh.StringResponse(USERDATA) + + data = self.datasource.crawl_metadata_service() + self.assertNotEqual(data, dict()) @mock.patch("cloudinit.url_helper.readurl") - def test_read_metadata_gets_malformed_response(self, m_readurl): + def test_read_malformed_metadata(self, m_readurl): def url_responses(url, **params): bad_json = json.dumps(METADATA)[:-2] - return url_helper.StringResponse(bad_json, code=200) + return uh.StringResponse(bad_json.encode(), code=200) m_readurl.side_effect = url_responses - with self.assertRaises(sources.InvalidMetaDataException): - ds_mod.read_metadata( - self.base_url, self.datasource.get_url_params() - ) - - @mock.patch("cloudinit.sources.DataSourceCloudCIX.read_metadata") - def test_not_on_cloudcix_returns_false(self, m_read_metadata): - self.m_read_dmi_data.return_value = "WrongCloud" - self.assertFalse(self.datasource.get_data()) - m_read_metadata.assert_not_called() + self.assertRaises( + InvalidMetaDataException, + ds_mod.read_metadata, + ds_mod.METADATA_URLS[0], + self.datasource.get_url_params(), + ) From b29a4a004dfd13d8fd127e94ef9604913e797285 Mon Sep 17 00:00:00 2001 From: Brian Date: Thu, 21 Mar 2024 17:23:10 +0000 Subject: [PATCH 18/32] Fix lint errors --- cloudinit/settings.py | 2 +- cloudinit/sources/DataSourceCloudCIX.py | 22 +++++++--------------- tests/unittests/sources/test_cloudcix.py | 9 ++++----- tests/unittests/sources/test_common.py | 2 +- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 70d885a8f90..34417c54e18 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -35,7 +35,6 @@ "AliYun", "Vultr", "Ec2", - "CloudCIX", "CloudSigma", "CloudStack", "SmartOS", @@ -51,6 +50,7 @@ "NWCS", "Akamai", "WSL", + "CloudCIX", # At the end to act as a 'catch' when none of the above work... "None", ], diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 6b281457d9c..ef22e35ad56 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -1,15 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. -import base64 import json import logging from typing import Optional -from requests.exceptions import ConnectionError -from cloudinit import atomic_helper, dmi, helpers -from cloudinit import net, sources, url_helper, util -from cloudinit.sources.helpers import ec2 + +from cloudinit import dmi, net, sources, url_helper, util from cloudinit.net.dhcp import NoDHCPLeaseError -from cloudinit.net.ephemeral import EphemeralDHCPv4, EphemeralIPNetwork +from cloudinit.net.ephemeral import EphemeralIPNetwork LOG = logging.getLogger(__name__) @@ -30,8 +27,6 @@ class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" - _metadata_url: str - def __init__(self, sys_cfg, distro, paths): super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) self.distro = distro @@ -63,7 +58,7 @@ def _get_data(self): ): crawled_data = util.log_time( logfunc=LOG.debug, - msg=f"Crawl of metadata service", + msg="Crawl of metadata service", func=self.crawl_metadata_service, ) except NoDHCPLeaseError as e: @@ -84,7 +79,7 @@ def crawl_metadata_service(self) -> dict: md_url = self.determine_md_url() if md_url is None: raise sources.InvalidMetaDataException( - f"Could not reach determine MetaData url" + "Could not reach determine MetaData url" ) data = read_metadata(md_url, self.get_url_params()) @@ -129,7 +124,7 @@ def determine_md_url(self) -> Optional[str]: @staticmethod def ds_detect(): - product_name = dmi.read_dmi_data("system-product") + product_name = dmi.read_dmi_data("system-product-name") return product_name == CLOUDCIX_DMI_NAME @property @@ -153,9 +148,6 @@ def _generate_net_cfg(self, metadata): "Metadata mac address %s not found.", iface["mac_address"] ) continue - interfaces["name"] = iface - - for name, iface in interfaces.items(): netcfg["ethernets"][name] = { "set-name": name, "match": { @@ -171,7 +163,7 @@ def read_metadata(base_url, url_params): md = {} leaf_key_format_callback = ( ("metadata", "meta-data", util.load_json), - ("userdata", "user-data", util.maybe_64decode), + ("userdata", "user-data", util.maybe_b64decode), ) for url_leaf, new_key, format_callback in leaf_key_format_callback: diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 8340d6648b2..62b4baa5a22 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,11 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. import json -from cloudinit import distros, helpers, sources, url_helper as uh -from cloudinit.sources import ( - DataSourceCloudCIX as ds_mod, - InvalidMetaDataException, -) +from cloudinit import distros, helpers, sources +from cloudinit import url_helper as uh +from cloudinit.sources import DataSourceCloudCIX as ds_mod +from cloudinit.sources import InvalidMetaDataException from tests.unittests.helpers import CiTestCase, mock METADATA = { diff --git a/tests/unittests/sources/test_common.py b/tests/unittests/sources/test_common.py index e00c808f96f..88d22b8dd08 100644 --- a/tests/unittests/sources/test_common.py +++ b/tests/unittests/sources/test_common.py @@ -62,6 +62,7 @@ VMware.DataSourceVMware, NWCS.DataSourceNWCS, Akamai.DataSourceAkamaiLocal, + CloudCIX.DataSourceCloudCIX, WSL.DataSourceWSL, ] @@ -69,7 +70,6 @@ AliYun.DataSourceAliYun, AltCloud.DataSourceAltCloud, Bigstep.DataSourceBigstep, - CloudCIX.DataSourceCloudCIX, CloudStack.DataSourceCloudStack, DSNone.DataSourceNone, Ec2.DataSourceEc2, From 657ffef4b801c410d4433d3464466be92403b23b Mon Sep 17 00:00:00 2001 From: Brian Date: Wed, 27 Mar 2024 15:05:15 +0000 Subject: [PATCH 19/32] Mock dhcp network for CloudCIX DS unittests --- cloudinit/sources/DataSourceCloudCIX.py | 9 +- tests/unittests/sources/test_cloudcix.py | 142 ++++++++++++++--------- 2 files changed, 93 insertions(+), 58 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index ef22e35ad56..380dcdea1b1 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -43,11 +43,8 @@ def __init__(self, sys_cfg, distro, paths): self.url_sec_between_retries = self.ds_cfg.get("wait") def _get_data(self): - """Fetch the user data, the metadata and the VM password - from the metadata service. - - Please refer to the datasource documentation for details on how the - metadata server and password server are crawled. + """ + Fetch the user data and the metadata """ try: with EphemeralIPNetwork( @@ -79,7 +76,7 @@ def crawl_metadata_service(self) -> dict: md_url = self.determine_md_url() if md_url is None: raise sources.InvalidMetaDataException( - "Could not reach determine MetaData url" + "Could not determine MetaData url" ) data = read_metadata(md_url, self.get_url_params()) diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 62b4baa5a22..72cf655e3c9 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,11 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. import json +import responses + from cloudinit import distros, helpers, sources from cloudinit import url_helper as uh +from cloudinit.net.ephemeral import EphemeralIPNetwork from cloudinit.sources import DataSourceCloudCIX as ds_mod from cloudinit.sources import InvalidMetaDataException -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import ResponsesTestCase, mock METADATA = { "instance_id": "12_34", @@ -28,35 +31,27 @@ }, } -USERDATA = b"""#cloud-config +USERDATA = """#cloud-config runcmd: - [ echo Hello, World >> /etc/greeting ] """ -def mock_imds(base_url, version): - # Create a function that mocks responses from a metadata server - version_str = f"v{version}" - - def func(url, **urlparams): - - if url == base_url: - return uh.StringResponse("Metadata enabled") - - md_endpoint = uh.combine_url(base_url, version_str, "metadata") - if url == md_endpoint: - return uh.StringResponse(json.dumps(METADATA).encode()) +class MockImds: + @staticmethod + def base_response(response): + return 200, response.headers, "Metadata enabled" - userdata_endpoint = uh.combine_url(base_url, version_str, "userdata") - if url == userdata_endpoint: - return uh.StringResponse(USERDATA) + @staticmethod + def metadata_response(response): + return 200, response.headers, json.dumps(METADATA).encode() - return uh.StringResponse("Not Found", code=404) + @staticmethod + def userdata_response(response): + return 200, response.headers, USERDATA.encode() - return func - -class TestDataSourceCloudCIX(CiTestCase): +class TestDataSourceCloudCIX(ResponsesTestCase): """ Test reading the meta-data """ @@ -71,6 +66,21 @@ def setUp(self): "m_read_dmi_data", return_value="CloudCIX", ) + self.add_patch( + "cloudinit.net.find_fallback_nic", + "_m_find_fallback_nic", + return_value="cixnic0", + ) + self.add_patch( + "cloudinit.sources.DataSourceCloudCIX.EphemeralIPNetwork", + "_m_EphemeralIPNetwork", + ) + + def noop(self): + return None + + self._m_EphemeralIPNetwork.__enter__ = noop + self._m_EphemeralIPNetwork.__exit__ = noop def _get_ds(self): distro_cls = distros.fetch("ubuntu") @@ -115,11 +125,20 @@ def test_setting_config_options(self): self.assertEqual(new_ds.url_timeout, cix_options["timeout"]) self.assertEqual(new_ds.url_retries, cix_options["retries"]) - @mock.patch("cloudinit.url_helper.readurl") - def test_determine_md_url(self, m_readurl): + def test_determine_md_url(self): base_url = ds_mod.METADATA_URLS[0] version = ds_mod.METADATA_VERSION - m_readurl.side_effect = mock_imds(base_url, version) + self.responses.reset() + self.responses.add_callback( + responses.GET, + base_url, + callback=MockImds.base_response, + ) + self.responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) md_url = self._get_ds().determine_md_url() @@ -129,28 +148,33 @@ def test_determine_md_url(self, m_readurl): ) self.assertEqual(md_url, expected_url) - @mock.patch("cloudinit.url_helper.readurl") - def test_reading_metadata_on_cloudcix(self, m_readurl): + def test_reading_metadata_on_cloudcix(self): base_url = ds_mod.METADATA_URLS[0] version = ds_mod.METADATA_VERSION - m_readurl.side_effect = mock_imds(base_url, version) + # Set up mock endpoints + self.responses.reset() + self.responses.add_callback( + responses.GET, + base_url, + callback=MockImds.base_response, + ) + self.responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) + self.responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "userdata"), + callback=MockImds.userdata_response, + ) self.assertTrue(self.datasource.get_data()) self.assertEqual(self.datasource.metadata, METADATA) - self.assertEqual(self.datasource.userdata_raw, USERDATA.decode()) - - @mock.patch("cloudinit.url_helper.readurl") - def test_failing_imds_endpoints(self, m_readurl): - # Set up an empty imds - endpoints = dict() - - def faulty_imds(url, **urlparams): - if url in endpoints: - return endpoints[url] - return uh.StringResponse(b"Not Found", code=404) - - m_readurl.side_effect = faulty_imds + self.assertEqual(self.datasource.userdata_raw, USERDATA) + def test_failing_imds_endpoints(self): + # Make request before imds is set up self.assertRaises( sources.InvalidMetaDataException, self.datasource.crawl_metadata_service, @@ -158,7 +182,11 @@ def faulty_imds(url, **urlparams): # Make imds respond to healthcheck base_url = ds_mod.METADATA_URLS[0] - endpoints[base_url] = uh.StringResponse("Metadata enabled") + self.responses.add_callback( + responses.GET, + base_url, + callback=MockImds.base_response, + ) self.assertRaises( sources.InvalidMetaDataException, @@ -166,9 +194,12 @@ def faulty_imds(url, **urlparams): ) # Make imds serve metadata - version_str = f"v{ds_mod.METADATA_VERSION}" - md_url = uh.combine_url(base_url, version_str, "metadata") - endpoints[md_url] = uh.StringResponse(json.dumps(METADATA).encode()) + version = ds_mod.METADATA_VERSION + self.responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) self.assertRaises( sources.InvalidMetaDataException, @@ -176,19 +207,26 @@ def faulty_imds(url, **urlparams): ) # Make imds serve userdata - ud_url = uh.combine_url(base_url, version_str, "userdata") - endpoints[ud_url] = uh.StringResponse(USERDATA) + self.responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "userdata"), + callback=MockImds.userdata_response, + ) data = self.datasource.crawl_metadata_service() self.assertNotEqual(data, dict()) - @mock.patch("cloudinit.url_helper.readurl") - def test_read_malformed_metadata(self, m_readurl): - def url_responses(url, **params): - bad_json = json.dumps(METADATA)[:-2] - return uh.StringResponse(bad_json.encode(), code=200) + def test_read_malformed_metadata(self): + def bad_response(response): + return 200, response.headers, json.dumps(METADATA)[:-2] - m_readurl.side_effect = url_responses + version = ds_mod.METADATA_VERSION + base_url = ds_mod.METADATA_URLS[0] + self.responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) self.assertRaises( InvalidMetaDataException, From f2314730f641c2e412299349ccb761a85eb692af Mon Sep 17 00:00:00 2001 From: Brian Date: Wed, 27 Mar 2024 16:50:44 +0000 Subject: [PATCH 20/32] Simplify Tests --- cloudinit/sources/DataSourceCloudCIX.py | 7 ++- doc/rtd/reference/datasources/cloudcix.rst | 2 +- tests/unittests/sources/test_cloudcix.py | 55 +++++++++++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 380dcdea1b1..778cff974a7 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -121,8 +121,7 @@ def determine_md_url(self) -> Optional[str]: @staticmethod def ds_detect(): - product_name = dmi.read_dmi_data("system-product-name") - return product_name == CLOUDCIX_DMI_NAME + return is_platform_viable() @property def network_config(self): @@ -156,6 +155,10 @@ def _generate_net_cfg(self, metadata): return netcfg +def is_platform_viable() -> bool: + return dmi.read_dmi_data("system-product-name") == CLOUDCIX_DMI_NAME + + def read_metadata(base_url, url_params): md = {} leaf_key_format_callback = ( diff --git a/doc/rtd/reference/datasources/cloudcix.rst b/doc/rtd/reference/datasources/cloudcix.rst index bba2c22baa2..97f4362aacb 100644 --- a/doc/rtd/reference/datasources/cloudcix.rst +++ b/doc/rtd/reference/datasources/cloudcix.rst @@ -7,7 +7,7 @@ CloudCIX serves metadata through an internal server, accessible at ``http://169.254.169.254/v1``. The metadata and userdata can be fetched at the ``/metadata`` and ``/userdata`` paths respectively. -CloudCIX instances are identified by the dmi Product Name. +CloudCIX instances are identified by the dmi product name `CloudCIX`. Configuration ------------- diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 72cf655e3c9..5830046f06f 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -5,10 +5,9 @@ from cloudinit import distros, helpers, sources from cloudinit import url_helper as uh -from cloudinit.net.ephemeral import EphemeralIPNetwork from cloudinit.sources import DataSourceCloudCIX as ds_mod from cloudinit.sources import InvalidMetaDataException -from tests.unittests.helpers import ResponsesTestCase, mock +from tests.unittests.helpers import ResponsesTestCase METADATA = { "instance_id": "12_34", @@ -72,15 +71,16 @@ def setUp(self): return_value="cixnic0", ) self.add_patch( - "cloudinit.sources.DataSourceCloudCIX.EphemeralIPNetwork", - "_m_EphemeralIPNetwork", + "cloudinit.net.ephemeral.EphemeralIPNetwork.__enter__", + "_m_EphemeralIPNetwork_enter", + return_value=None, + ) + self.add_patch( + "cloudinit.net.ephemeral.EphemeralIPNetwork.__exit__", + "_m_EphemeralIPNetwork_exit", + return_value=None, ) - def noop(self): - return None - - self._m_EphemeralIPNetwork.__enter__ = noop - self._m_EphemeralIPNetwork.__exit__ = noop def _get_ds(self): distro_cls = distros.fetch("ubuntu") @@ -101,9 +101,11 @@ def _get_ds(self): def test_identifying_cloudcix(self): self.assertTrue(self.datasource.ds_detect()) + self.assertTrue(ds_mod.is_platform_viable()) self.m_read_dmi_data.return_value = "OnCloud9" self.assertFalse(self.datasource.ds_detect()) + self.assertFalse(ds_mod.is_platform_viable()) def test_setting_config_options(self): cix_options = { @@ -222,15 +224,44 @@ def bad_response(response): version = ds_mod.METADATA_VERSION base_url = ds_mod.METADATA_URLS[0] + versioned_url = uh.combine_url(base_url, f"v{version}") + + # Malformed metadata self.responses.add_callback( responses.GET, - uh.combine_url(base_url, f"v{version}", "metadata"), - callback=MockImds.metadata_response, + uh.combine_url(versioned_url, "metadata"), + callback=bad_response, + ) + self.responses.add_callback( + responses.GET, + uh.combine_url(versioned_url, "userdata"), + callback=MockImds.userdata_response, ) self.assertRaises( InvalidMetaDataException, ds_mod.read_metadata, - ds_mod.METADATA_URLS[0], + versioned_url, + self.datasource.get_url_params(), + ) + + def test_bad_response_code(self): + def bad_response(response): + return 404, response.headers, "" + + version = ds_mod.METADATA_VERSION + base_url = ds_mod.METADATA_URLS[0] + versioned_url = uh.combine_url(base_url, f"v{version}") + + self.responses.add_callback( + responses.GET, + uh.combine_url(versioned_url, "metadata"), + callback=bad_response, + ) + + self.assertRaises( + InvalidMetaDataException, + ds_mod.read_metadata, + versioned_url, self.datasource.get_url_params(), ) From a765050bf36a55c632425575781ba26c05f67854 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Wed, 21 Aug 2024 13:19:47 +0200 Subject: [PATCH 21/32] Address review comments on #1351 This commits implements most of blackboxsw's review suggestions from 2024-04-04. Only one missing for now is the decode_binary/maybe_b64decode one. --- cloudinit/sources/DataSourceCloudCIX.py | 33 ++++++++-------------- doc/rtd/reference/datasources/cloudcix.rst | 6 ++-- tools/.github-cla-signers | 1 + 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 778cff974a7..878c8697e4b 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -13,34 +13,22 @@ METADATA_URLS = ["http://169.254.169.254"] METADATA_VERSION = 1 -BUILTIN_DS_CONFIG = { - "retries": 3, - "timeout": 5, - "wait": 5, -} - - CLOUDCIX_DMI_NAME = "CloudCIX" class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" + # Setup read_url parameters through get_url_params() + url_retries = 3 + url_timeout = 5 + url_sec_between_retries = 5 def __init__(self, sys_cfg, distro, paths): super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) self.distro = distro - self.ds_cfg = util.mergemanydict( - [ - util.get_cfg_by_path(sys_cfg, ["datasource", "CloudCIX"], {}), - BUILTIN_DS_CONFIG, - ] - ) self._metadata_url = None self._net_cfg = None - self.url_retries = self.ds_cfg.get("retries") - self.url_timeout = self.ds_cfg.get("timeout") - self.url_sec_between_retries = self.ds_cfg.get("wait") def _get_data(self): """ @@ -52,12 +40,13 @@ def _get_data(self): interface=net.find_fallback_nic(), ipv6=True, ipv4=True, - ): - crawled_data = util.log_time( - logfunc=LOG.debug, - msg="Crawl of metadata service", - func=self.crawl_metadata_service, - ) + ) as netw: + state_msg = f" {netw.state_msg}" if netw.state_msg else "" + crawled_data = util.log_time( + logfunc=LOG.debug, + msg=f"Crawl of metadata service{state_msg}", + func=self.crawl_metadata_service, + ) except NoDHCPLeaseError as e: LOG.error("Bailing, DHCP exception: %s", e) return False diff --git a/doc/rtd/reference/datasources/cloudcix.rst b/doc/rtd/reference/datasources/cloudcix.rst index 97f4362aacb..a05a8d0b96c 100644 --- a/doc/rtd/reference/datasources/cloudcix.rst +++ b/doc/rtd/reference/datasources/cloudcix.rst @@ -3,7 +3,7 @@ CloudCIX ======== -CloudCIX serves metadata through an internal server, accessible at +`CloudCIX`_ serves metadata through an internal server, accessible at ``http://169.254.169.254/v1``. The metadata and userdata can be fetched at the ``/metadata`` and ``/userdata`` paths respectively. @@ -20,7 +20,7 @@ CloudCIX datasource has the following config options: CloudCIX: retries: 3 timeout: 2 - wait_retry: 2 + sec_between_retries: 2 - *retries*: The number of times the datasource should try to connect to the @@ -30,4 +30,4 @@ CloudCIX datasource has the following config options: - *wait*: How long in seconds to wait between consecutive requests to the metadata service -.. vi: textwidth=78 +_CloudCIX: https://www.cloudcix.com/ diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 32d6eaade5b..720d5924c7e 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -92,6 +92,7 @@ jcmoore3 Jehops jf jfroche +jgrassler Jille jinkkkang JohnKepplers From 9469fe2d49661aa0ca6a0bf446713733b8c17760 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Wed, 21 Aug 2024 15:07:52 +0200 Subject: [PATCH 22/32] Fix test breakage from addressing review comments --- cloudinit/sources/DataSourceCloudCIX.py | 18 +++++++++--------- tests/unittests/sources/test_cloudcix.py | 24 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 878c8697e4b..ef46a0f5bf9 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -21,7 +21,7 @@ class DataSourceCloudCIX(sources.DataSource): dsname = "CloudCIX" # Setup read_url parameters through get_url_params() url_retries = 3 - url_timeout = 5 + url_timeout_seconds = 5 url_sec_between_retries = 5 def __init__(self, sys_cfg, distro, paths): @@ -35,18 +35,18 @@ def _get_data(self): Fetch the user data and the metadata """ try: - with EphemeralIPNetwork( + netw = EphemeralIPNetwork( self.distro, interface=net.find_fallback_nic(), ipv6=True, ipv4=True, - ) as netw: - state_msg = f" {netw.state_msg}" if netw.state_msg else "" - crawled_data = util.log_time( - logfunc=LOG.debug, - msg=f"Crawl of metadata service{state_msg}", - func=self.crawl_metadata_service, - ) + ) + state_msg = f" {netw.state_msg}" if netw.state_msg else "" + crawled_data = util.log_time( + logfunc=LOG.debug, + msg=f"Crawl of metadata service{state_msg}", + func=self.crawl_metadata_service, + ) except NoDHCPLeaseError as e: LOG.error("Bailing, DHCP exception: %s", e) return False diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 5830046f06f..01c39dd4257 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -50,6 +50,12 @@ def userdata_response(response): return 200, response.headers, USERDATA.encode() +class MockEphemeralIPNetworkWithStateMsg: + @property + def state_msg(self): + return "Mock state" + + class TestDataSourceCloudCIX(ResponsesTestCase): """ Test reading the meta-data @@ -73,15 +79,14 @@ def setUp(self): self.add_patch( "cloudinit.net.ephemeral.EphemeralIPNetwork.__enter__", "_m_EphemeralIPNetwork_enter", - return_value=None, + return_value=MockEphemeralIPNetworkWithStateMsg(), ) self.add_patch( "cloudinit.net.ephemeral.EphemeralIPNetwork.__exit__", "_m_EphemeralIPNetwork_exit", - return_value=None, + return_value=MockEphemeralIPNetworkWithStateMsg(), ) - def _get_ds(self): distro_cls = distros.fetch("ubuntu") distro = distro_cls("ubuntu", cfg={}, paths=self.paths) @@ -111,6 +116,7 @@ def test_setting_config_options(self): cix_options = { "timeout": 1234, "retries": 5678, + "sec_between_retries": 9012, } sys_cfg = { "datasource": { @@ -124,8 +130,16 @@ def test_setting_config_options(self): new_ds = ds_mod.DataSourceCloudCIX( sys_cfg=sys_cfg, distro=distro, paths=self.paths ) - self.assertEqual(new_ds.url_timeout, cix_options["timeout"]) - self.assertEqual(new_ds.url_retries, cix_options["retries"]) + self.assertEqual( + new_ds.get_url_params().timeout_seconds, cix_options["timeout"] + ) + self.assertEqual( + new_ds.get_url_params().num_retries, cix_options["retries"] + ) + self.assertEqual( + new_ds.get_url_params().sec_between_retries, + cix_options["sec_between_retries"], + ) def test_determine_md_url(self): base_url = ds_mod.METADATA_URLS[0] From 5e4d2ed3be2c16626065fe8281296f2e49c56713 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Wed, 21 Aug 2024 15:21:15 +0200 Subject: [PATCH 23/32] Add missing _unpickle for CloudCIX data source --- cloudinit/sources/DataSourceCloudCIX.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index ef46a0f5bf9..3d0bc1cdb9f 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -30,6 +30,13 @@ def __init__(self, sys_cfg, distro, paths): self._metadata_url = None self._net_cfg = None + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + if not hasattr(self, "_metadata_url"): + setattr(self, "_metadata_url", None) + if not hasattr(self, "_net_cfg"): + setattr(self, "_net_cfg", None) + def _get_data(self): """ Fetch the user data and the metadata From aed1adaebddb6c572575fbc04e5d260ce3c57343 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Thu, 22 Aug 2024 13:49:30 +0000 Subject: [PATCH 24/32] Add NETWORK dependency to CloudCIX data source. --- cloudinit/sources/DataSourceCloudCIX.py | 2 +- tests/unittests/sources/test_common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 3d0bc1cdb9f..a1759be24c2 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -193,7 +193,7 @@ def read_metadata(base_url, url_params): # Used to match classes to dependencies datasources = [ - (DataSourceCloudCIX, (sources.DEP_FILESYSTEM,)), + (DataSourceCloudCIX, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/tests/unittests/sources/test_common.py b/tests/unittests/sources/test_common.py index 5b3339fd89d..4f6e847bd5c 100644 --- a/tests/unittests/sources/test_common.py +++ b/tests/unittests/sources/test_common.py @@ -62,7 +62,6 @@ VMware.DataSourceVMware, NWCS.DataSourceNWCS, Akamai.DataSourceAkamaiLocal, - CloudCIX.DataSourceCloudCIX, WSL.DataSourceWSL, ] @@ -83,6 +82,7 @@ UpCloud.DataSourceUpCloud, Akamai.DataSourceAkamai, VMware.DataSourceVMware, + CloudCIX.DataSourceCloudCIX, ] From 5ebac09e15f4da80fb968275e0d826ae83c47962 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Fri, 30 Aug 2024 18:00:24 +0000 Subject: [PATCH 25/32] Make various checks happy * Fix missing TOC entry for cloudcix.rst * Sort .github-cla-signers correctly * Fix mypy complaints --- cloudinit/sources/DataSourceCloudCIX.py | 4 ++-- doc/rtd/reference/datasources.rst | 1 + tests/unittests/sources/test_cloudcix.py | 5 ++++- tools/.github-cla-signers | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index a1759be24c2..af10dfccba2 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -2,7 +2,7 @@ import json import logging -from typing import Optional +from typing import Any, Dict, Optional from cloudinit import dmi, net, sources, url_helper, util from cloudinit.net.dhcp import NoDHCPLeaseError @@ -130,7 +130,7 @@ def network_config(self): return self._net_cfg def _generate_net_cfg(self, metadata): - netcfg = {"version": 2, "ethernets": {}} + netcfg: Dict[str, Any] = {"version": 2, "ethernets": {}} macs_to_nics = net.get_interfaces_by_mac() for iface in metadata["network"]["interfaces"]: diff --git a/doc/rtd/reference/datasources.rst b/doc/rtd/reference/datasources.rst index 9826fa56314..d195b9b75dd 100644 --- a/doc/rtd/reference/datasources.rst +++ b/doc/rtd/reference/datasources.rst @@ -43,6 +43,7 @@ The following is a list of documentation for each supported datasource: datasources/altcloud.rst datasources/ec2.rst datasources/azure.rst + datasources/cloudcix.rst datasources/cloudsigma.rst datasources/cloudstack.rst datasources/configdrive.rst diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 01c39dd4257..d573744db71 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -2,6 +2,7 @@ import json import responses +from unittest import mock from cloudinit import distros, helpers, sources from cloudinit import url_helper as uh @@ -60,12 +61,14 @@ class TestDataSourceCloudCIX(ResponsesTestCase): """ Test reading the meta-data """ + allowed_subp = True def setUp(self): super(TestDataSourceCloudCIX, self).setUp() self.paths = helpers.Paths({"run_dir": self.tmp_dir()}) self.datasource = self._get_ds() - self.allowed_subp = True + self.m_read_dmi_data = mock.MagicMock() + self.add_patch( "cloudinit.dmi.read_dmi_data", "m_read_dmi_data", diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 720d5924c7e..bd29dcb562a 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -27,9 +27,9 @@ berolinux bin456789 bipinbachhao BirknerAlex -BrinKe-dev bmhughes brianphaley +BrinKe-dev CalvoM candlerb CarlosNihelton From 61d90419a14383baafc6bd1e3192bb51529af633 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Mon, 2 Sep 2024 15:34:01 +0000 Subject: [PATCH 26/32] Converted test_cloudcix.py to pytest. --- tests/unittests/sources/test_cloudcix.py | 118 ++++++++++++----------- 1 file changed, 63 insertions(+), 55 deletions(-) diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index d573744db71..876358f04b2 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. import json +from unittest.mock import PropertyMock +import pytest import responses -from unittest import mock from cloudinit import distros, helpers, sources from cloudinit import url_helper as uh from cloudinit.sources import DataSourceCloudCIX as ds_mod from cloudinit.sources import InvalidMetaDataException -from tests.unittests.helpers import ResponsesTestCase METADATA = { "instance_id": "12_34", @@ -57,37 +57,41 @@ def state_msg(self): return "Mock state" -class TestDataSourceCloudCIX(ResponsesTestCase): +class TestDataSourceCloudCIX: """ Test reading the meta-data """ + allowed_subp = True - def setUp(self): - super(TestDataSourceCloudCIX, self).setUp() - self.paths = helpers.Paths({"run_dir": self.tmp_dir()}) + @pytest.fixture(autouse=True) + def setup(self, mocker, tmp_path): + self.paths = helpers.Paths({"run_dir": tmp_path}) self.datasource = self._get_ds() - self.m_read_dmi_data = mock.MagicMock() - - self.add_patch( + self.m_read_dmi_data = mocker.patch( "cloudinit.dmi.read_dmi_data", - "m_read_dmi_data", - return_value="CloudCIX", + new_callable=PropertyMock, ) - self.add_patch( + self.m_read_dmi_data.return_value = "CloudCIX" + + self._m_find_fallback_nic = mocker.patch( "cloudinit.net.find_fallback_nic", - "_m_find_fallback_nic", - return_value="cixnic0", + new_callable=PropertyMock, ) - self.add_patch( + self._m_find_fallback_nic.return_value = "cixnic0" + self._m_EphemeralIPNetwork_enter = mocker.patch( "cloudinit.net.ephemeral.EphemeralIPNetwork.__enter__", - "_m_EphemeralIPNetwork_enter", - return_value=MockEphemeralIPNetworkWithStateMsg(), + new_callable=PropertyMock, ) - self.add_patch( + self._m_EphemeralIPNetwork_enter.return_value = ( + MockEphemeralIPNetworkWithStateMsg() + ) + self._m_EphemeralIPNetwork_exit = mocker.patch( "cloudinit.net.ephemeral.EphemeralIPNetwork.__exit__", - "_m_EphemeralIPNetwork_exit", - return_value=MockEphemeralIPNetworkWithStateMsg(), + new_callable=PropertyMock, + ) + self._m_EphemeralIPNetwork_exit.return_value = ( + MockEphemeralIPNetworkWithStateMsg() ) def _get_ds(self): @@ -107,13 +111,14 @@ def _get_ds(self): paths=self.paths, ) + @responses.activate def test_identifying_cloudcix(self): - self.assertTrue(self.datasource.ds_detect()) - self.assertTrue(ds_mod.is_platform_viable()) + assert self.datasource.ds_detect() + assert ds_mod.is_platform_viable() self.m_read_dmi_data.return_value = "OnCloud9" - self.assertFalse(self.datasource.ds_detect()) - self.assertFalse(ds_mod.is_platform_viable()) + assert not self.datasource.ds_detect() + assert not ds_mod.is_platform_viable() def test_setting_config_options(self): cix_options = { @@ -133,27 +138,26 @@ def test_setting_config_options(self): new_ds = ds_mod.DataSourceCloudCIX( sys_cfg=sys_cfg, distro=distro, paths=self.paths ) - self.assertEqual( - new_ds.get_url_params().timeout_seconds, cix_options["timeout"] - ) - self.assertEqual( - new_ds.get_url_params().num_retries, cix_options["retries"] + assert ( + new_ds.get_url_params().timeout_seconds == cix_options["timeout"] ) - self.assertEqual( - new_ds.get_url_params().sec_between_retries, - cix_options["sec_between_retries"], + assert new_ds.get_url_params().num_retries == cix_options["retries"] + assert ( + new_ds.get_url_params().sec_between_retries + == cix_options["sec_between_retries"] ) + @responses.activate def test_determine_md_url(self): base_url = ds_mod.METADATA_URLS[0] version = ds_mod.METADATA_VERSION - self.responses.reset() - self.responses.add_callback( + responses.reset() + responses.add_callback( responses.GET, base_url, callback=MockImds.base_response, ) - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "metadata"), callback=MockImds.metadata_response, @@ -165,76 +169,79 @@ def test_determine_md_url(self): ds_mod.METADATA_URLS[0], f"v{version}", ) - self.assertEqual(md_url, expected_url) + assert md_url == expected_url + @responses.activate def test_reading_metadata_on_cloudcix(self): base_url = ds_mod.METADATA_URLS[0] version = ds_mod.METADATA_VERSION # Set up mock endpoints - self.responses.reset() - self.responses.add_callback( + responses.reset() + responses.add_callback( responses.GET, base_url, callback=MockImds.base_response, ) - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "metadata"), callback=MockImds.metadata_response, ) - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "userdata"), callback=MockImds.userdata_response, ) - self.assertTrue(self.datasource.get_data()) - self.assertEqual(self.datasource.metadata, METADATA) - self.assertEqual(self.datasource.userdata_raw, USERDATA) + assert self.datasource.get_data() + assert self.datasource.metadata == METADATA + assert self.datasource.userdata_raw == USERDATA + @responses.activate def test_failing_imds_endpoints(self): # Make request before imds is set up - self.assertRaises( + pytest.raises( sources.InvalidMetaDataException, self.datasource.crawl_metadata_service, ) # Make imds respond to healthcheck base_url = ds_mod.METADATA_URLS[0] - self.responses.add_callback( + responses.add_callback( responses.GET, base_url, callback=MockImds.base_response, ) - self.assertRaises( + pytest.raises( sources.InvalidMetaDataException, self.datasource.crawl_metadata_service, ) # Make imds serve metadata version = ds_mod.METADATA_VERSION - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "metadata"), callback=MockImds.metadata_response, ) - self.assertRaises( + pytest.raises( sources.InvalidMetaDataException, self.datasource.crawl_metadata_service, ) # Make imds serve userdata - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "userdata"), callback=MockImds.userdata_response, ) data = self.datasource.crawl_metadata_service() - self.assertNotEqual(data, dict()) + assert data != dict() + @responses.activate def test_read_malformed_metadata(self): def bad_response(response): return 200, response.headers, json.dumps(METADATA)[:-2] @@ -244,24 +251,25 @@ def bad_response(response): versioned_url = uh.combine_url(base_url, f"v{version}") # Malformed metadata - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(versioned_url, "metadata"), callback=bad_response, ) - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(versioned_url, "userdata"), callback=MockImds.userdata_response, ) - self.assertRaises( + pytest.raises( InvalidMetaDataException, ds_mod.read_metadata, versioned_url, self.datasource.get_url_params(), ) + @responses.activate def test_bad_response_code(self): def bad_response(response): return 404, response.headers, "" @@ -270,13 +278,13 @@ def bad_response(response): base_url = ds_mod.METADATA_URLS[0] versioned_url = uh.combine_url(base_url, f"v{version}") - self.responses.add_callback( + responses.add_callback( responses.GET, uh.combine_url(versioned_url, "metadata"), callback=bad_response, ) - self.assertRaises( + pytest.raises( InvalidMetaDataException, ds_mod.read_metadata, versioned_url, From 7d42f490c2b2be716e59404b0f279804462fd746 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Mon, 9 Sep 2024 07:30:23 +0000 Subject: [PATCH 27/32] Addressed second round of review comments * Applied suggested diff * Removed EphemeralIPNetwork setup * Removed redundant local variable from determine_md_url() * Check for actual error message in invalid JSON unit test --- cloudinit/sources/DataSourceCloudCIX.py | 37 +++++------ tests/unittests/sources/test_cloudcix.py | 84 +++++++++++++----------- 2 files changed, 64 insertions(+), 57 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index af10dfccba2..cf23ff06278 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -5,8 +5,6 @@ from typing import Any, Dict, Optional from cloudinit import dmi, net, sources, url_helper, util -from cloudinit.net.dhcp import NoDHCPLeaseError -from cloudinit.net.ephemeral import EphemeralIPNetwork LOG = logging.getLogger(__name__) @@ -26,7 +24,6 @@ class DataSourceCloudCIX(sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) - self.distro = distro self._metadata_url = None self._net_cfg = None @@ -42,21 +39,11 @@ def _get_data(self): Fetch the user data and the metadata """ try: - netw = EphemeralIPNetwork( - self.distro, - interface=net.find_fallback_nic(), - ipv6=True, - ipv4=True, - ) - state_msg = f" {netw.state_msg}" if netw.state_msg else "" crawled_data = util.log_time( logfunc=LOG.debug, - msg=f"Crawl of metadata service{state_msg}", + msg="Crawl of metadata service", func=self.crawl_metadata_service, ) - except NoDHCPLeaseError as e: - LOG.error("Bailing, DHCP exception: %s", e) - return False except sources.InvalidMetaDataException as error: LOG.debug( "Failed to read data from CloudCIX datasource: %s", error @@ -72,7 +59,7 @@ def crawl_metadata_service(self) -> dict: md_url = self.determine_md_url() if md_url is None: raise sources.InvalidMetaDataException( - "Could not determine MetaData url" + "Could not determine metadata URL" ) data = read_metadata(md_url, self.get_url_params()) @@ -93,7 +80,6 @@ def determine_md_url(self) -> Optional[str]: return None # Find the highest supported metadata version - md_url = None for version in range(METADATA_VERSION, 0, -1): url = url_helper.combine_url( base_url, "v{0}".format(version), "metadata" @@ -105,14 +91,13 @@ def determine_md_url(self) -> Optional[str]: continue if response.ok(): - md_url = url_helper.combine_url( + self._metadata_url = url_helper.combine_url( base_url, "v{0}".format(version) ) break else: LOG.debug("No metadata found at URL %s", url) - self._metadata_url = md_url return self._metadata_url @staticmethod @@ -137,7 +122,7 @@ def _generate_net_cfg(self, metadata): name = macs_to_nics.get(iface["mac_address"]) if name is None: LOG.warning( - "Metadata mac address %s not found.", iface["mac_address"] + "Metadata MAC address %s not found.", iface["mac_address"] ) continue netcfg["ethernets"][name] = { @@ -155,7 +140,19 @@ def is_platform_viable() -> bool: return dmi.read_dmi_data("system-product-name") == CLOUDCIX_DMI_NAME -def read_metadata(base_url, url_params): +def read_metadata(base_url: str, url_params): + """ + Read metadata from metadata server at base_url + + :returns: dictionary of retrieved metadata and user data containing the + following keys: meta-data, user-data + :param: base_url: meta data server's base URL + :param: url_params: dictionary of URL retrieval parameters. Valid keys are + `retries`, `sec_between` and `timeout`. + :raises: InvalidMetadataException upon network error connecting to metadata + URL, error response from meta data server or failure to + decode/parse metadata and userdata payload. + """ md = {} leaf_key_format_callback = ( ("metadata", "meta-data", util.load_json), diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 876358f04b2..2d6602e5a28 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -1,11 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. import json -from unittest.mock import PropertyMock +from unittest import mock import pytest import responses -from cloudinit import distros, helpers, sources +from cloudinit import distros, sources from cloudinit import url_helper as uh from cloudinit.sources import DataSourceCloudCIX as ds_mod from cloudinit.sources import InvalidMetaDataException @@ -62,33 +62,31 @@ class TestDataSourceCloudCIX: Test reading the meta-data """ - allowed_subp = True - @pytest.fixture(autouse=True) - def setup(self, mocker, tmp_path): - self.paths = helpers.Paths({"run_dir": tmp_path}) + def setup(self, mocker, tmpdir, paths): + self.paths = paths self.datasource = self._get_ds() self.m_read_dmi_data = mocker.patch( "cloudinit.dmi.read_dmi_data", - new_callable=PropertyMock, + new_callable=mock.PropertyMock, ) self.m_read_dmi_data.return_value = "CloudCIX" self._m_find_fallback_nic = mocker.patch( "cloudinit.net.find_fallback_nic", - new_callable=PropertyMock, + new_callable=mock.PropertyMock, ) self._m_find_fallback_nic.return_value = "cixnic0" self._m_EphemeralIPNetwork_enter = mocker.patch( "cloudinit.net.ephemeral.EphemeralIPNetwork.__enter__", - new_callable=PropertyMock, + new_callable=mock.PropertyMock, ) self._m_EphemeralIPNetwork_enter.return_value = ( MockEphemeralIPNetworkWithStateMsg() ) self._m_EphemeralIPNetwork_exit = mocker.patch( "cloudinit.net.ephemeral.EphemeralIPNetwork.__exit__", - new_callable=PropertyMock, + new_callable=mock.PropertyMock, ) self._m_EphemeralIPNetwork_exit.return_value = ( MockEphemeralIPNetworkWithStateMsg() @@ -198,40 +196,46 @@ def test_reading_metadata_on_cloudcix(self): assert self.datasource.userdata_raw == USERDATA @responses.activate - def test_failing_imds_endpoints(self): + def test_failing_imds_endpoints(self, mocker): + sleep = mocker.patch("time.sleep") + base_url = ds_mod.METADATA_URLS[0] # Make request before imds is set up - pytest.raises( + with pytest.raises( sources.InvalidMetaDataException, - self.datasource.crawl_metadata_service, - ) + match="Could not determine metadata URL", + ): + self.datasource.crawl_metadata_service() - # Make imds respond to healthcheck - base_url = ds_mod.METADATA_URLS[0] + # Make imds respond to healthcheck but fail v1/metadata responses.add_callback( responses.GET, base_url, callback=MockImds.base_response, ) - - pytest.raises( + version = ds_mod.METADATA_VERSION + with pytest.raises( sources.InvalidMetaDataException, - self.datasource.crawl_metadata_service, - ) + match="Could not determine metadata URL", + ): + self.datasource.crawl_metadata_service() - # Make imds serve metadata - version = ds_mod.METADATA_VERSION + # No sleep/retries when md_url returns 404. No viable IMDS found. + assert 0 == sleep.call_count + + # Make imds serve metadata but ConnectionError on userdata responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "metadata"), callback=MockImds.metadata_response, ) - pytest.raises( sources.InvalidMetaDataException, self.datasource.crawl_metadata_service, ) + # Sleep called number of default datasource configured "retries" + assert [mock.call(5)] == sleep.call_args_list - # Make imds serve userdata + # Make IMDS serve userdata responses.add_callback( responses.GET, uh.combine_url(base_url, f"v{version}", "userdata"), @@ -239,7 +243,10 @@ def test_failing_imds_endpoints(self): ) data = self.datasource.crawl_metadata_service() - assert data != dict() + assert data == { + "meta-data": METADATA, + "user-data": USERDATA.encode("utf-8"), + } @responses.activate def test_read_malformed_metadata(self): @@ -262,15 +269,16 @@ def bad_response(response): callback=MockImds.userdata_response, ) - pytest.raises( + with pytest.raises( InvalidMetaDataException, - ds_mod.read_metadata, - versioned_url, - self.datasource.get_url_params(), - ) + match="Invalid JSON at http://169.254.169.254/v1/metadata", + ): + ds_mod.read_metadata( + versioned_url, self.datasource.get_url_params(), + ) @responses.activate - def test_bad_response_code(self): + def test_bad_response_code(self, mocker): def bad_response(response): return 404, response.headers, "" @@ -283,10 +291,12 @@ def bad_response(response): uh.combine_url(versioned_url, "metadata"), callback=bad_response, ) - - pytest.raises( + sleep = mocker.patch("time.sleep") + with pytest.raises( InvalidMetaDataException, - ds_mod.read_metadata, - versioned_url, - self.datasource.get_url_params(), - ) + match=f"Failed to fetch IMDS metadata: {versioned_url}", + ): + ds_mod.read_metadata( + versioned_url, self.datasource.get_url_params() + ) + assert [mock.call(5)] == sleep.call_args_list From 6dce91ebe64ce8120885f0b136c85c21a99af235 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Tue, 10 Sep 2024 07:24:09 +0000 Subject: [PATCH 28/32] Fix sec_between_retries item in CloudCIX data source documentation. --- doc/rtd/reference/datasources/cloudcix.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/rtd/reference/datasources/cloudcix.rst b/doc/rtd/reference/datasources/cloudcix.rst index a05a8d0b96c..9bd9a083cc7 100644 --- a/doc/rtd/reference/datasources/cloudcix.rst +++ b/doc/rtd/reference/datasources/cloudcix.rst @@ -27,7 +27,7 @@ CloudCIX datasource has the following config options: metadata service - *timeout*: How long in seconds to wait for a response from the metadata service -- *wait*: How long in seconds to wait between consecutive requests to the - metadata service +- *sec_between_retries*: How long in seconds to wait between consecutive + requests to the metadata service _CloudCIX: https://www.cloudcix.com/ From 8a890c290ffbd8aee0f531123a356a1f0f48ceb6 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Tue, 10 Sep 2024 15:14:54 +0000 Subject: [PATCH 29/32] Add routes and DNS config to metadata. --- cloudinit/sources/DataSourceCloudCIX.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index cf23ff06278..10fa617ba84 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -117,9 +117,11 @@ def network_config(self): def _generate_net_cfg(self, metadata): netcfg: Dict[str, Any] = {"version": 2, "ethernets": {}} macs_to_nics = net.get_interfaces_by_mac() + nameservers = metadata["network"].get("nameservers", None) for iface in metadata["network"]["interfaces"]: name = macs_to_nics.get(iface["mac_address"]) + routes = iface.get("routes", None) if name is None: LOG.warning( "Metadata MAC address %s not found.", iface["mac_address"] @@ -133,6 +135,11 @@ def _generate_net_cfg(self, metadata): "addresses": iface["addresses"], } + if nameservers is not None: + netcfg["ethernets"][name]["nameservers"] = nameservers + if routes is not None: + netcfg["ethernets"][name]["routes"] = routes + return netcfg From daacae77e07e2df24106a3d49013b1a308835c73 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Tue, 10 Sep 2024 18:36:43 +0000 Subject: [PATCH 30/32] Adjusted unit tests * Added an expected network configuration that should be produced by the metadata used for tests. * Added an extra assert in test_reading_metadata_on_cloudcix to make sure the network configuration generated from metadata matches the expected network configuration. * Removed mocks for EphemeralIPNetwork --- tests/unittests/sources/test_cloudcix.py | 76 +++++++++++++++++------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index 2d6602e5a28..cdba4129f44 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -7,15 +7,18 @@ from cloudinit import distros, sources from cloudinit import url_helper as uh +from cloudinit.atomic_helper import json_dumps from cloudinit.sources import DataSourceCloudCIX as ds_mod from cloudinit.sources import InvalidMetaDataException METADATA = { "instance_id": "12_34", "network": { + "nameservers": {"addresses": ["10.0.0.1"], "search": ["cloudcix.com"]}, "interfaces": [ { "mac_address": "ab:cd:ef:00:01:02", + "routes": [{"to": "default", "via": "10.0.0.1"}], "addresses": [ "10.0.0.2/24", "192.168.0.2/24", @@ -27,7 +30,42 @@ "10.10.10.2/24", ], }, - ] + ], + }, +} + +# Expected network config resulting from METADATA +NETWORK_CONFIG = { + "version": 2, + "ethernets": { + "enp1s0": { + "set-name": "enp1s0", + "addresses": [ + "10.0.0.2/24", + "192.168.0.2/24", + ], + "match": {"macaddress": "ab:cd:ef:00:01:02"}, + "nameservers": { + "addresses": ["10.0.0.1"], + "search": [ + "cloudcix.com", + ], + }, + "routes": [{"to": "default", "via": "10.0.0.1"}], + }, + "enp2s0": { + "set-name": "enp2s0", + "addresses": [ + "10.10.10.2/24", + ], + "match": {"macaddress": "12:34:56:ab:cd:ef"}, + "nameservers": { + "addresses": ["10.0.0.1"], + "search": [ + "cloudcix.com", + ], + }, + }, }, } @@ -51,12 +89,6 @@ def userdata_response(response): return 200, response.headers, USERDATA.encode() -class MockEphemeralIPNetworkWithStateMsg: - @property - def state_msg(self): - return "Mock state" - - class TestDataSourceCloudCIX: """ Test reading the meta-data @@ -77,20 +109,14 @@ def setup(self, mocker, tmpdir, paths): new_callable=mock.PropertyMock, ) self._m_find_fallback_nic.return_value = "cixnic0" - self._m_EphemeralIPNetwork_enter = mocker.patch( - "cloudinit.net.ephemeral.EphemeralIPNetwork.__enter__", + self._m_get_interfaces_by_mac = mocker.patch( + "cloudinit.net.get_interfaces_by_mac", new_callable=mock.PropertyMock, ) - self._m_EphemeralIPNetwork_enter.return_value = ( - MockEphemeralIPNetworkWithStateMsg() - ) - self._m_EphemeralIPNetwork_exit = mocker.patch( - "cloudinit.net.ephemeral.EphemeralIPNetwork.__exit__", - new_callable=mock.PropertyMock, - ) - self._m_EphemeralIPNetwork_exit.return_value = ( - MockEphemeralIPNetworkWithStateMsg() - ) + self._m_get_interfaces_by_mac.return_value = { + "ab:cd:ef:00:01:02": "enp1s0", + "12:34:56:ab:cd:ef": "enp2s0", + } def _get_ds(self): distro_cls = distros.fetch("ubuntu") @@ -194,6 +220,9 @@ def test_reading_metadata_on_cloudcix(self): assert self.datasource.get_data() assert self.datasource.metadata == METADATA assert self.datasource.userdata_raw == USERDATA + assert json_dumps( + self.datasource._generate_net_cfg(METADATA) + ) == json_dumps(NETWORK_CONFIG) @responses.activate def test_failing_imds_endpoints(self, mocker): @@ -272,10 +301,11 @@ def bad_response(response): with pytest.raises( InvalidMetaDataException, match="Invalid JSON at http://169.254.169.254/v1/metadata", - ): - ds_mod.read_metadata( - versioned_url, self.datasource.get_url_params(), - ) + ): + ds_mod.read_metadata( + versioned_url, + self.datasource.get_url_params(), + ) @responses.activate def test_bad_response_code(self, mocker): From 88cdc11e53e8022a515fd996913c141abc300fa9 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Tue, 17 Sep 2024 16:18:10 +0000 Subject: [PATCH 31/32] Pass complete netplan configuration data structure. --- cloudinit/sources/DataSourceCloudCIX.py | 36 ++---------------- tests/unittests/sources/test_cloudcix.py | 48 ++++++++++++------------ 2 files changed, 29 insertions(+), 55 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 10fa617ba84..6371434eee2 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -2,9 +2,9 @@ import json import logging -from typing import Any, Dict, Optional +from typing import Optional -from cloudinit import dmi, net, sources, url_helper, util +from cloudinit import dmi, sources, url_helper, util LOG = logging.getLogger(__name__) @@ -45,7 +45,7 @@ def _get_data(self): func=self.crawl_metadata_service, ) except sources.InvalidMetaDataException as error: - LOG.debug( + LOG.error( "Failed to read data from CloudCIX datasource: %s", error ) return False @@ -111,37 +111,9 @@ def network_config(self): if not self.metadata: return None - self._net_cfg = self._generate_net_cfg(self.metadata) + self._net_cfg = self.metadata["network"] return self._net_cfg - def _generate_net_cfg(self, metadata): - netcfg: Dict[str, Any] = {"version": 2, "ethernets": {}} - macs_to_nics = net.get_interfaces_by_mac() - nameservers = metadata["network"].get("nameservers", None) - - for iface in metadata["network"]["interfaces"]: - name = macs_to_nics.get(iface["mac_address"]) - routes = iface.get("routes", None) - if name is None: - LOG.warning( - "Metadata MAC address %s not found.", iface["mac_address"] - ) - continue - netcfg["ethernets"][name] = { - "set-name": name, - "match": { - "macaddress": iface["mac_address"].lower(), - }, - "addresses": iface["addresses"], - } - - if nameservers is not None: - netcfg["ethernets"][name]["nameservers"] = nameservers - if routes is not None: - netcfg["ethernets"][name]["routes"] = routes - - return netcfg - def is_platform_viable() -> bool: return dmi.read_dmi_data("system-product-name") == CLOUDCIX_DMI_NAME diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py index cdba4129f44..531e6022f9f 100644 --- a/tests/unittests/sources/test_cloudcix.py +++ b/tests/unittests/sources/test_cloudcix.py @@ -14,23 +14,33 @@ METADATA = { "instance_id": "12_34", "network": { - "nameservers": {"addresses": ["10.0.0.1"], "search": ["cloudcix.com"]}, - "interfaces": [ - { - "mac_address": "ab:cd:ef:00:01:02", - "routes": [{"to": "default", "via": "10.0.0.1"}], + "version": 2, + "ethernets": { + "eth0": { + "set-name": "eth0", + "match": {"macaddress": "ab:cd:ef:00:01:02"}, "addresses": [ "10.0.0.2/24", "192.168.0.2/24", ], + "nameservers": { + "addresses": ["10.0.0.1"], + "search": ["cloudcix.com"], + }, + "routes": [{"to": "default", "via": "10.0.0.1"}], }, - { - "mac_address": "12:34:56:ab:cd:ef", + "eth1": { + "set-name": "eth1", + "match": {"macaddress": "12:34:56:ab:cd:ef"}, "addresses": [ "10.10.10.2/24", ], + "nameservers": { + "addresses": ["10.0.0.1"], + "search": ["cloudcix.com"], + }, }, - ], + }, }, } @@ -38,8 +48,8 @@ NETWORK_CONFIG = { "version": 2, "ethernets": { - "enp1s0": { - "set-name": "enp1s0", + "eth0": { + "set-name": "eth0", "addresses": [ "10.0.0.2/24", "192.168.0.2/24", @@ -53,8 +63,8 @@ }, "routes": [{"to": "default", "via": "10.0.0.1"}], }, - "enp2s0": { - "set-name": "enp2s0", + "eth1": { + "set-name": "eth1", "addresses": [ "10.10.10.2/24", ], @@ -109,14 +119,6 @@ def setup(self, mocker, tmpdir, paths): new_callable=mock.PropertyMock, ) self._m_find_fallback_nic.return_value = "cixnic0" - self._m_get_interfaces_by_mac = mocker.patch( - "cloudinit.net.get_interfaces_by_mac", - new_callable=mock.PropertyMock, - ) - self._m_get_interfaces_by_mac.return_value = { - "ab:cd:ef:00:01:02": "enp1s0", - "12:34:56:ab:cd:ef": "enp2s0", - } def _get_ds(self): distro_cls = distros.fetch("ubuntu") @@ -220,9 +222,9 @@ def test_reading_metadata_on_cloudcix(self): assert self.datasource.get_data() assert self.datasource.metadata == METADATA assert self.datasource.userdata_raw == USERDATA - assert json_dumps( - self.datasource._generate_net_cfg(METADATA) - ) == json_dumps(NETWORK_CONFIG) + assert json_dumps(self.datasource.network_config) == json_dumps( + NETWORK_CONFIG + ) @responses.activate def test_failing_imds_endpoints(self, mocker): From 68bcb339050e5663f96f87037ea93e1dc02b6776 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Thu, 19 Sep 2024 07:40:27 +0000 Subject: [PATCH 32/32] Removed _unpickle and test broken by its lack. --- cloudinit/sources/DataSourceCloudCIX.py | 7 ------- tests/unittests/test_upgrade.py | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py index 6371434eee2..8f6ef4a1b6f 100644 --- a/cloudinit/sources/DataSourceCloudCIX.py +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -27,13 +27,6 @@ def __init__(self, sys_cfg, distro, paths): self._metadata_url = None self._net_cfg = None - def _unpickle(self, ci_pkl_version: int) -> None: - super()._unpickle(ci_pkl_version) - if not hasattr(self, "_metadata_url"): - setattr(self, "_metadata_url", None) - if not hasattr(self, "_net_cfg"): - setattr(self, "_net_cfg", None) - def _get_data(self): """ Fetch the user data and the metadata diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index 5c8eef5a582..32a3d7c2ba3 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -47,6 +47,7 @@ class TestUpgrade: "seed", "seed_dir", }, + "CloudCIX": {"_metadata_url", "_net_cfg"}, "CloudSigma": {"cepko", "ssh_public_key"}, "CloudStack": { "api_ver",