diff --git a/cloudinit/apport.py b/cloudinit/apport.py index 8d16db9cf78..3bc1055caad 100644 --- a/cloudinit/apport.py +++ b/cloudinit/apport.py @@ -36,6 +36,7 @@ "Azure", "Bigstep", "Brightbox", + "CloudCIX", "CloudSigma", "CloudStack", "DigitalOcean", diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 6d98d079543..a73f25118d7 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -50,6 +50,7 @@ "NWCS", "Akamai", "WSL", + "CloudCIX", # At the end to act as a 'catch' when none of the above work... "None", ], diff --git a/cloudinit/sources/DataSourceCloudCIX.py b/cloudinit/sources/DataSourceCloudCIX.py new file mode 100644 index 00000000000..8f6ef4a1b6f --- /dev/null +++ b/cloudinit/sources/DataSourceCloudCIX.py @@ -0,0 +1,171 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import json +import logging +from typing import Optional + +from cloudinit import dmi, sources, url_helper, util + +LOG = logging.getLogger(__name__) + +METADATA_URLS = ["http://169.254.169.254"] +METADATA_VERSION = 1 + +CLOUDCIX_DMI_NAME = "CloudCIX" + + +class DataSourceCloudCIX(sources.DataSource): + + dsname = "CloudCIX" + # Setup read_url parameters through get_url_params() + url_retries = 3 + url_timeout_seconds = 5 + url_sec_between_retries = 5 + + def __init__(self, sys_cfg, distro, paths): + super(DataSourceCloudCIX, self).__init__(sys_cfg, distro, paths) + self._metadata_url = None + self._net_cfg = None + + def _get_data(self): + """ + Fetch the user data and the metadata + """ + try: + crawled_data = util.log_time( + logfunc=LOG.debug, + msg="Crawl of metadata service", + func=self.crawl_metadata_service, + ) + except sources.InvalidMetaDataException as error: + LOG.error( + "Failed to read data from CloudCIX datasource: %s", error + ) + return False + + self.metadata = crawled_data["meta-data"] + self.userdata_raw = util.decode_binary(crawled_data["user-data"]) + + return True + + def crawl_metadata_service(self) -> dict: + md_url = self.determine_md_url() + if md_url is None: + raise sources.InvalidMetaDataException( + "Could not determine metadata URL" + ) + + data = read_metadata(md_url, self.get_url_params()) + return data + + def determine_md_url(self) -> Optional[str]: + if self._metadata_url: + return self._metadata_url + + # Try to reach the metadata server + url_params = self.get_url_params() + base_url, _ = url_helper.wait_for_url( + METADATA_URLS, + max_wait=url_params.max_wait_seconds, + timeout=url_params.timeout_seconds, + ) + if not base_url: + return None + + # Find the highest supported metadata version + for version in range(METADATA_VERSION, 0, -1): + url = url_helper.combine_url( + base_url, "v{0}".format(version), "metadata" + ) + try: + response = url_helper.readurl(url, timeout=self.url_timeout) + except url_helper.UrlError as e: + LOG.debug("URL %s raised exception %s", url, e) + continue + + if response.ok(): + self._metadata_url = url_helper.combine_url( + base_url, "v{0}".format(version) + ) + break + else: + LOG.debug("No metadata found at URL %s", url) + + return self._metadata_url + + @staticmethod + def ds_detect(): + return is_platform_viable() + + @property + def network_config(self): + if self._net_cfg: + return self._net_cfg + + if not self.metadata: + return None + self._net_cfg = self.metadata["network"] + return self._net_cfg + + +def is_platform_viable() -> bool: + return dmi.read_dmi_data("system-product-name") == CLOUDCIX_DMI_NAME + + +def read_metadata(base_url: str, url_params): + """ + Read metadata from metadata server at base_url + + :returns: dictionary of retrieved metadata and user data containing the + following keys: meta-data, user-data + :param: base_url: meta data server's base URL + :param: url_params: dictionary of URL retrieval parameters. Valid keys are + `retries`, `sec_between` and `timeout`. + :raises: InvalidMetadataException upon network error connecting to metadata + URL, error response from meta data server or failure to + decode/parse metadata and userdata payload. + """ + md = {} + leaf_key_format_callback = ( + ("metadata", "meta-data", util.load_json), + ("userdata", "user-data", util.maybe_b64decode), + ) + + for url_leaf, new_key, format_callback in leaf_key_format_callback: + try: + response = url_helper.readurl( + url=url_helper.combine_url(base_url, url_leaf), + retries=url_params.num_retries, + sec_between=url_params.sec_between_retries, + timeout=url_params.timeout_seconds, + ) + except url_helper.UrlError as error: + raise sources.InvalidMetaDataException( + f"Failed to fetch IMDS {url_leaf}: " + f"{base_url}/{url_leaf}: {error}" + ) + + if not response.ok(): + raise sources.InvalidMetaDataException( + f"No valid {url_leaf} found. " + f"URL {base_url}/{url_leaf} returned code {response.code}" + ) + + try: + md[new_key] = format_callback(response.contents) + except json.decoder.JSONDecodeError as exc: + raise sources.InvalidMetaDataException( + f"Invalid JSON at {base_url}/{url_leaf}: {exc}" + ) from exc + return md + + +# Used to match classes to dependencies +datasources = [ + (DataSourceCloudCIX, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f404d1130bd..3306b1069ce 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -268,9 +268,7 @@ def __init__(self, contents, url, code=200): self.url = url def ok(self, *args, **kwargs): - if self.code != 200: - return False - return True + return self.code == 200 def __str__(self): return self.contents.decode("utf-8") diff --git a/doc/rtd/reference/datasources.rst b/doc/rtd/reference/datasources.rst index 9826fa56314..d195b9b75dd 100644 --- a/doc/rtd/reference/datasources.rst +++ b/doc/rtd/reference/datasources.rst @@ -43,6 +43,7 @@ The following is a list of documentation for each supported datasource: datasources/altcloud.rst datasources/ec2.rst datasources/azure.rst + datasources/cloudcix.rst datasources/cloudsigma.rst datasources/cloudstack.rst datasources/configdrive.rst diff --git a/doc/rtd/reference/datasources/cloudcix.rst b/doc/rtd/reference/datasources/cloudcix.rst new file mode 100644 index 00000000000..9bd9a083cc7 --- /dev/null +++ b/doc/rtd/reference/datasources/cloudcix.rst @@ -0,0 +1,33 @@ +.. _datasource_cloudcix: + +CloudCIX +======== + +`CloudCIX`_ serves metadata through an internal server, accessible at +``http://169.254.169.254/v1``. The metadata and userdata can be fetched at +the ``/metadata`` and ``/userdata`` paths respectively. + +CloudCIX instances are identified by the dmi product name `CloudCIX`. + +Configuration +------------- + +CloudCIX datasource has the following config options: + +:: + + datasource: + CloudCIX: + retries: 3 + timeout: 2 + sec_between_retries: 2 + + +- *retries*: The number of times the datasource should try to connect to the + metadata service +- *timeout*: How long in seconds to wait for a response from the metadata + service +- *sec_between_retries*: How long in seconds to wait between consecutive + requests to the metadata service + +_CloudCIX: https://www.cloudcix.com/ diff --git a/tests/unittests/sources/test_cloudcix.py b/tests/unittests/sources/test_cloudcix.py new file mode 100644 index 00000000000..531e6022f9f --- /dev/null +++ b/tests/unittests/sources/test_cloudcix.py @@ -0,0 +1,334 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import json +from unittest import mock + +import pytest +import responses + +from cloudinit import distros, sources +from cloudinit import url_helper as uh +from cloudinit.atomic_helper import json_dumps +from cloudinit.sources import DataSourceCloudCIX as ds_mod +from cloudinit.sources import InvalidMetaDataException + +METADATA = { + "instance_id": "12_34", + "network": { + "version": 2, + "ethernets": { + "eth0": { + "set-name": "eth0", + "match": {"macaddress": "ab:cd:ef:00:01:02"}, + "addresses": [ + "10.0.0.2/24", + "192.168.0.2/24", + ], + "nameservers": { + "addresses": ["10.0.0.1"], + "search": ["cloudcix.com"], + }, + "routes": [{"to": "default", "via": "10.0.0.1"}], + }, + "eth1": { + "set-name": "eth1", + "match": {"macaddress": "12:34:56:ab:cd:ef"}, + "addresses": [ + "10.10.10.2/24", + ], + "nameservers": { + "addresses": ["10.0.0.1"], + "search": ["cloudcix.com"], + }, + }, + }, + }, +} + +# Expected network config resulting from METADATA +NETWORK_CONFIG = { + "version": 2, + "ethernets": { + "eth0": { + "set-name": "eth0", + "addresses": [ + "10.0.0.2/24", + "192.168.0.2/24", + ], + "match": {"macaddress": "ab:cd:ef:00:01:02"}, + "nameservers": { + "addresses": ["10.0.0.1"], + "search": [ + "cloudcix.com", + ], + }, + "routes": [{"to": "default", "via": "10.0.0.1"}], + }, + "eth1": { + "set-name": "eth1", + "addresses": [ + "10.10.10.2/24", + ], + "match": {"macaddress": "12:34:56:ab:cd:ef"}, + "nameservers": { + "addresses": ["10.0.0.1"], + "search": [ + "cloudcix.com", + ], + }, + }, + }, +} + +USERDATA = """#cloud-config +runcmd: +- [ echo Hello, World >> /etc/greeting ] +""" + + +class MockImds: + @staticmethod + def base_response(response): + return 200, response.headers, "Metadata enabled" + + @staticmethod + def metadata_response(response): + return 200, response.headers, json.dumps(METADATA).encode() + + @staticmethod + def userdata_response(response): + return 200, response.headers, USERDATA.encode() + + +class TestDataSourceCloudCIX: + """ + Test reading the meta-data + """ + + @pytest.fixture(autouse=True) + def setup(self, mocker, tmpdir, paths): + self.paths = paths + self.datasource = self._get_ds() + self.m_read_dmi_data = mocker.patch( + "cloudinit.dmi.read_dmi_data", + new_callable=mock.PropertyMock, + ) + self.m_read_dmi_data.return_value = "CloudCIX" + + self._m_find_fallback_nic = mocker.patch( + "cloudinit.net.find_fallback_nic", + new_callable=mock.PropertyMock, + ) + self._m_find_fallback_nic.return_value = "cixnic0" + + def _get_ds(self): + distro_cls = distros.fetch("ubuntu") + distro = distro_cls("ubuntu", cfg={}, paths=self.paths) + return ds_mod.DataSourceCloudCIX( + sys_cfg={ + "datasource": { + "CloudCIX": { + "retries": 1, + "timeout": 0, + "wait": 0, + }, + } + }, + distro=distro, + paths=self.paths, + ) + + @responses.activate + def test_identifying_cloudcix(self): + assert self.datasource.ds_detect() + assert ds_mod.is_platform_viable() + + self.m_read_dmi_data.return_value = "OnCloud9" + assert not self.datasource.ds_detect() + assert not ds_mod.is_platform_viable() + + def test_setting_config_options(self): + cix_options = { + "timeout": 1234, + "retries": 5678, + "sec_between_retries": 9012, + } + sys_cfg = { + "datasource": { + "CloudCIX": cix_options, + } + } + + # Instantiate a new datasource + distro_cls = distros.fetch("ubuntu") + distro = distro_cls("ubuntu", cfg={}, paths=self.paths) + new_ds = ds_mod.DataSourceCloudCIX( + sys_cfg=sys_cfg, distro=distro, paths=self.paths + ) + assert ( + new_ds.get_url_params().timeout_seconds == cix_options["timeout"] + ) + assert new_ds.get_url_params().num_retries == cix_options["retries"] + assert ( + new_ds.get_url_params().sec_between_retries + == cix_options["sec_between_retries"] + ) + + @responses.activate + def test_determine_md_url(self): + base_url = ds_mod.METADATA_URLS[0] + version = ds_mod.METADATA_VERSION + responses.reset() + responses.add_callback( + responses.GET, + base_url, + callback=MockImds.base_response, + ) + responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) + + md_url = self._get_ds().determine_md_url() + + expected_url = uh.combine_url( + ds_mod.METADATA_URLS[0], + f"v{version}", + ) + assert md_url == expected_url + + @responses.activate + def test_reading_metadata_on_cloudcix(self): + base_url = ds_mod.METADATA_URLS[0] + version = ds_mod.METADATA_VERSION + # Set up mock endpoints + responses.reset() + responses.add_callback( + responses.GET, + base_url, + callback=MockImds.base_response, + ) + responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) + responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "userdata"), + callback=MockImds.userdata_response, + ) + + assert self.datasource.get_data() + assert self.datasource.metadata == METADATA + assert self.datasource.userdata_raw == USERDATA + assert json_dumps(self.datasource.network_config) == json_dumps( + NETWORK_CONFIG + ) + + @responses.activate + def test_failing_imds_endpoints(self, mocker): + sleep = mocker.patch("time.sleep") + base_url = ds_mod.METADATA_URLS[0] + # Make request before imds is set up + with pytest.raises( + sources.InvalidMetaDataException, + match="Could not determine metadata URL", + ): + self.datasource.crawl_metadata_service() + + # Make imds respond to healthcheck but fail v1/metadata + responses.add_callback( + responses.GET, + base_url, + callback=MockImds.base_response, + ) + version = ds_mod.METADATA_VERSION + with pytest.raises( + sources.InvalidMetaDataException, + match="Could not determine metadata URL", + ): + self.datasource.crawl_metadata_service() + + # No sleep/retries when md_url returns 404. No viable IMDS found. + assert 0 == sleep.call_count + + # Make imds serve metadata but ConnectionError on userdata + responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "metadata"), + callback=MockImds.metadata_response, + ) + pytest.raises( + sources.InvalidMetaDataException, + self.datasource.crawl_metadata_service, + ) + # Sleep called number of default datasource configured "retries" + assert [mock.call(5)] == sleep.call_args_list + + # Make IMDS serve userdata + responses.add_callback( + responses.GET, + uh.combine_url(base_url, f"v{version}", "userdata"), + callback=MockImds.userdata_response, + ) + + data = self.datasource.crawl_metadata_service() + assert data == { + "meta-data": METADATA, + "user-data": USERDATA.encode("utf-8"), + } + + @responses.activate + def test_read_malformed_metadata(self): + def bad_response(response): + return 200, response.headers, json.dumps(METADATA)[:-2] + + version = ds_mod.METADATA_VERSION + base_url = ds_mod.METADATA_URLS[0] + versioned_url = uh.combine_url(base_url, f"v{version}") + + # Malformed metadata + responses.add_callback( + responses.GET, + uh.combine_url(versioned_url, "metadata"), + callback=bad_response, + ) + responses.add_callback( + responses.GET, + uh.combine_url(versioned_url, "userdata"), + callback=MockImds.userdata_response, + ) + + with pytest.raises( + InvalidMetaDataException, + match="Invalid JSON at http://169.254.169.254/v1/metadata", + ): + ds_mod.read_metadata( + versioned_url, + self.datasource.get_url_params(), + ) + + @responses.activate + def test_bad_response_code(self, mocker): + def bad_response(response): + return 404, response.headers, "" + + version = ds_mod.METADATA_VERSION + base_url = ds_mod.METADATA_URLS[0] + versioned_url = uh.combine_url(base_url, f"v{version}") + + responses.add_callback( + responses.GET, + uh.combine_url(versioned_url, "metadata"), + callback=bad_response, + ) + sleep = mocker.patch("time.sleep") + with pytest.raises( + InvalidMetaDataException, + match=f"Failed to fetch IMDS metadata: {versioned_url}", + ): + ds_mod.read_metadata( + versioned_url, self.datasource.get_url_params() + ) + assert [mock.call(5)] == sleep.call_args_list diff --git a/tests/unittests/sources/test_common.py b/tests/unittests/sources/test_common.py index c35bff9b837..4f6e847bd5c 100644 --- a/tests/unittests/sources/test_common.py +++ b/tests/unittests/sources/test_common.py @@ -9,6 +9,7 @@ from cloudinit.sources import DataSourceAltCloud as AltCloud from cloudinit.sources import DataSourceAzure as Azure from cloudinit.sources import DataSourceBigstep as Bigstep +from cloudinit.sources import DataSourceCloudCIX as CloudCIX from cloudinit.sources import DataSourceCloudSigma as CloudSigma from cloudinit.sources import DataSourceCloudStack as CloudStack from cloudinit.sources import DataSourceConfigDrive as ConfigDrive @@ -81,6 +82,7 @@ UpCloud.DataSourceUpCloud, Akamai.DataSourceAkamai, VMware.DataSourceVMware, + CloudCIX.DataSourceCloudCIX, ] diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index d8f10c1ab8f..9aaf91156a3 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -1079,6 +1079,10 @@ def test_hetzner_found(self): """Hetzner cloud is identified in sys_vendor.""" self._test_ds_found("Hetzner") + def test_cloudcix_found(self): + """CloudCIX cloud is identified in dmi product-name""" + self._test_ds_found("CloudCIX") + def test_nwcs_found(self): """NWCS is identified in sys_vendor.""" self._test_ds_found("NWCS") @@ -1608,6 +1612,10 @@ def _print_run_output(rc, out, err, cfg, files): os.path.join(P_SEED_DIR, "azure", "ovf-env.xml"): "present\n", }, }, + "CloudCIX": { + "ds": "CloudCIX", + "files": {P_PRODUCT_NAME: "CloudCIX\n"}, + }, "Azure-parse-invalid": { "ds": "Azure", "files": { diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index 5c8eef5a582..32a3d7c2ba3 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -47,6 +47,7 @@ class TestUpgrade: "seed", "seed_dir", }, + "CloudCIX": {"_metadata_url", "_net_cfg"}, "CloudSigma": {"cepko", "ssh_public_key"}, "CloudStack": { "api_ver", diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 80872dcb508..bd29dcb562a 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -29,6 +29,7 @@ bipinbachhao BirknerAlex bmhughes brianphaley +BrinKe-dev CalvoM candlerb CarlosNihelton @@ -91,6 +92,7 @@ jcmoore3 Jehops jf jfroche +jgrassler Jille jinkkkang JohnKepplers diff --git a/tools/ds-identify b/tools/ds-identify index 606be9c4da5..e00b05e8096 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -131,7 +131,7 @@ DI_DSNAME="" DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ CloudSigma CloudStack DigitalOcean Vultr AliYun Ec2 GCE OpenNebula OpenStack \ OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud VMware \ -LXD NWCS Akamai WSL" +LXD NWCS Akamai WSL CloudCIX" DI_DSLIST="" DI_MODE="" DI_ON_FOUND="" @@ -791,6 +791,11 @@ dscheck_CloudStack() { return $DS_NOT_FOUND } +dscheck_CloudCIX() { + dmi_product_name_matches "CloudCIX" && return $DS_FOUND + return $DS_NOT_FOUND +} + dscheck_Exoscale() { dmi_product_name_matches "Exoscale*" && return $DS_FOUND return $DS_NOT_FOUND