From 7c63e8086e3e66f7e4b7f07a615693a7808e429f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 4 Feb 2022 14:06:15 -0700 Subject: [PATCH 1/3] cloud-id: publish /run/cloud-init/cloud-id- files Once a valid datasource is detected, publish the following artifacts to expedite cloud-identification without having to invoke cloud-id from shell scripts or sheling out from python. These files can also be relied on in systemd ConditionPathExists directives to limit execution of services and units to specific clouds. /run/cloud-init/cloud-id: - A symlink with content that is the canonical cloud-id of the datasource detected. This content is the same lower-case value as the output of /usr/bin/cloud-id. /run/cloud-init/cloud-id-: - A single file which will contain the canonical cloud-id encoded in the filename --- cloudinit/sources/__init__.py | 13 ++++++ .../modules/test_combined.py | 13 ++++++ tests/unittests/sources/test_init.py | 46 ++++++++++++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index bd862cdd149..2d19cbe56df 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -303,6 +303,9 @@ def _get_standardized_metadata(self, instance_data): "_beta_keys": ["subplatform"], "availability-zone": availability_zone, "availability_zone": availability_zone, + "cloud_id": canonical_cloud_id( + self.cloud_name, self.region, self.platform_type + ), "cloud-name": self.cloud_name, "cloud_name": self.cloud_name, "distro": sysinfo["dist"][0], @@ -408,6 +411,16 @@ def persist_instance_data(self): json_sensitive_file = os.path.join( self.paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE ) + cloud_id = instance_data["v1"].get("cloud_id", "none") + cloud_id_file = os.path.join(self.paths.run_dir, "cloud-id") + util.write_file(f"{cloud_id_file}-{cloud_id}", cloud_id) + if os.path.exists(cloud_id_file): + prev_cloud_id_file = os.path.realpath(cloud_id_file) + else: + prev_cloud_id_file = cloud_id_file + util.sym_link(f"{cloud_id_file}-{cloud_id}", cloud_id_file, force=True) + if prev_cloud_id_file != cloud_id_file: + util.del_file(prev_cloud_id_file) write_json(json_sensitive_file, processed_data, mode=0o600) json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE) # World readable diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 43aa809e6c5..b49e160b74b 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -249,6 +249,10 @@ def test_instance_json_lxd(self, class_client: IntegrationInstance): v1_data = data["v1"] assert v1_data["cloud_name"] == "unknown" assert v1_data["platform"] == "lxd" + assert v1_data["cloud_id"] == "lxd" + assert f"{v1_data['cloud_id']}\n" == client.read_from_file( + "/run/cloud-init/cloud-id-lxd" + ) assert ( v1_data["subplatform"] == "seed-dir (/var/lib/cloud/seed/nocloud-net)" @@ -270,6 +274,10 @@ def test_instance_json_lxd_vm(self, class_client: IntegrationInstance): v1_data = data["v1"] assert v1_data["cloud_name"] == "unknown" assert v1_data["platform"] == "lxd" + assert v1_data["cloud_id"] == "lxd" + assert f"{v1_data['cloud_id']}\n" == client.read_from_file( + "/run/cloud-init/cloud-id-lxd" + ) assert any( [ "/var/lib/cloud/seed/nocloud-net" in v1_data["subplatform"], @@ -291,6 +299,11 @@ def test_instance_json_ec2(self, class_client: IntegrationInstance): v1_data = data["v1"] assert v1_data["cloud_name"] == "aws" assert v1_data["platform"] == "ec2" + # Different regions will show up as ec2-(gov|china) + assert v1_data["cloud_id"].startswith("ec2") + assert f"{v1_data['cloud_id']}\n" == client.read_from_file( + "/run/cloud-init/cloud-id-ec2" + ) assert v1_data["subplatform"].startswith("metadata") assert ( v1_data["availability_zone"] == client.instance.availability_zone diff --git a/tests/unittests/sources/test_init.py b/tests/unittests/sources/test_init.py index 745a7fa6b63..a266a148bf0 100644 --- a/tests/unittests/sources/test_init.py +++ b/tests/unittests/sources/test_init.py @@ -378,7 +378,11 @@ def test_get_data_writes_json_instance_data_on_success(self): "dist": ["ubuntu", "20.04", "focal"], } with mock.patch("cloudinit.util.system_info", return_value=sys_info): - datasource.get_data() + with mock.patch( + "cloudinit.sources.canonical_cloud_id", + return_value="canonical_cloud_id", + ): + datasource.get_data() json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) content = util.load_file(json_file) expected = { @@ -390,6 +394,7 @@ def test_get_data_writes_json_instance_data_on_success(self): "_beta_keys": ["subplatform"], "availability-zone": "myaz", "availability_zone": "myaz", + "cloud_id": "canonical_cloud_id", "cloud-name": "subclasscloudname", "cloud_name": "subclasscloudname", "distro": "ubuntu", @@ -562,7 +567,11 @@ def test_get_data_writes_json_instance_data_sensitive(self): datasource.sensitive_metadata_keys, ) with mock.patch("cloudinit.util.system_info", return_value=sys_info): - datasource.get_data() + with mock.patch( + "cloudinit.sources.canonical_cloud_id", + return_value="canonical-cloud-id", + ): + datasource.get_data() sensitive_json_file = self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, tmp) content = util.load_file(sensitive_json_file) expected = { @@ -583,6 +592,7 @@ def test_get_data_writes_json_instance_data_sensitive(self): "_beta_keys": ["subplatform"], "availability-zone": "myaz", "availability_zone": "myaz", + "cloud_id": "canonical-cloud-id", "cloud-name": "subclasscloudname", "cloud_name": "subclasscloudname", "distro": "ubuntu", @@ -666,6 +676,38 @@ def test_persist_instance_data_writes_ec2_metadata_when_set(self): {"ec2stuff": "is good"}, instance_data["ds"]["ec2_metadata"] ) + def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): + """canonical-cloud-id class attribute is set, persist to json.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({"run_dir": tmp}) + ) + cloud_id_link = os.path.join(tmp, "cloud-id") + cloud_id_file = os.path.join(tmp, "cloud-id-my-cloud") + cloud_id2_file = os.path.join(tmp, "cloud-id-my-cloud2") + for filename in (cloud_id_file, cloud_id_link, cloud_id2_file): + self.assertFalse( + os.path.exists(filename), "Unexpected link found {filename}" + ) + with mock.patch( + "cloudinit.sources.canonical_cloud_id", return_value="my-cloud" + ): + datasource.get_data() + self.assertEqual("my-cloud", util.load_file(cloud_id_link)) + # A symlink with the generic /run/cloud-init/cloud-id link is present + self.assertTrue(util.is_link(cloud_id_link)) + # When cloud-id changes, symlink and content change + with mock.patch( + "cloudinit.sources.canonical_cloud_id", return_value="my-cloud2" + ): + datasource.persist_instance_data() + self.assertEqual("my-cloud2", util.load_file(cloud_id2_file)) + # Previous cloud-id- file removed + self.assertFalse(os.path.exists(cloud_id_file)) + # Generic link persisted which contains canonical-cloud-id as content + self.assertTrue(util.is_link(cloud_id_link)) + self.assertEqual("my-cloud2", util.load_file(cloud_id_link)) + def test_persist_instance_data_writes_network_json_when_set(self): """When network_data.json class attribute is set, persist to json.""" tmp = self.tmp_dir() From 5731a015f510bc8e6a63bdc2174de0f1ee704187 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 9 Feb 2022 20:12:39 -0700 Subject: [PATCH 2/3] tests: assert symlink for /run/cloud-init/cloud-id to specific cloud-id --- tests/integration_tests/modules/test_combined.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index b49e160b74b..2c57e7c3384 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -222,6 +222,16 @@ def test_correct_datasource_detected( == parsed_datasource ) + def test_cloud_id_file_symlink(self, class_client: IntegrationInstance): + cloud_id = class_client.execute("cloud-id").stdout + expected_link_output = ( + "'/run/cloud-init/cloud-id' -> " + f"'/run/cloud-init/cloud-id-{cloud_id}'" + ) + assert expected_link_output == str( + class_client.execute("stat -c %N /run/cloud-init/cloud-id") + ) + def _check_common_metadata(self, data): assert data["base64_encoded_keys"] == [] assert data["merged_cfg"] == "redacted for non-root user" From 13ec57803d71eb76757d925b8a39a511bd69fe59 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 10 Feb 2022 12:01:47 -0700 Subject: [PATCH 3/3] cloud-id: include trailing newline in /run/cloud-init/cloud-id instnace.read_from_file rstrips white space to integration tests will expect dropped trailing newline. --- cloudinit/sources/__init__.py | 2 +- tests/integration_tests/modules/test_combined.py | 9 ++++++--- tests/unittests/sources/test_init.py | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 2d19cbe56df..88028cfa6e9 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -413,7 +413,7 @@ def persist_instance_data(self): ) cloud_id = instance_data["v1"].get("cloud_id", "none") cloud_id_file = os.path.join(self.paths.run_dir, "cloud-id") - util.write_file(f"{cloud_id_file}-{cloud_id}", cloud_id) + util.write_file(f"{cloud_id_file}-{cloud_id}", f"{cloud_id}\n") if os.path.exists(cloud_id_file): prev_cloud_id_file = os.path.realpath(cloud_id_file) else: diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 2c57e7c3384..7a9a6e27fcf 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -260,7 +260,7 @@ def test_instance_json_lxd(self, class_client: IntegrationInstance): assert v1_data["cloud_name"] == "unknown" assert v1_data["platform"] == "lxd" assert v1_data["cloud_id"] == "lxd" - assert f"{v1_data['cloud_id']}\n" == client.read_from_file( + assert f"{v1_data['cloud_id']}" == client.read_from_file( "/run/cloud-init/cloud-id-lxd" ) assert ( @@ -285,7 +285,7 @@ def test_instance_json_lxd_vm(self, class_client: IntegrationInstance): assert v1_data["cloud_name"] == "unknown" assert v1_data["platform"] == "lxd" assert v1_data["cloud_id"] == "lxd" - assert f"{v1_data['cloud_id']}\n" == client.read_from_file( + assert f"{v1_data['cloud_id']}" == client.read_from_file( "/run/cloud-init/cloud-id-lxd" ) assert any( @@ -311,7 +311,7 @@ def test_instance_json_ec2(self, class_client: IntegrationInstance): assert v1_data["platform"] == "ec2" # Different regions will show up as ec2-(gov|china) assert v1_data["cloud_id"].startswith("ec2") - assert f"{v1_data['cloud_id']}\n" == client.read_from_file( + assert f"{v1_data['cloud_id']}" == client.read_from_file( "/run/cloud-init/cloud-id-ec2" ) assert v1_data["subplatform"].startswith("metadata") @@ -333,6 +333,9 @@ def test_instance_json_gce(self, class_client: IntegrationInstance): v1_data = data["v1"] assert v1_data["cloud_name"] == "gce" assert v1_data["platform"] == "gce" + assert f"{v1_data['cloud_id']}" == client.read_from_file( + "/run/cloud-init/cloud-id-gce" + ) assert v1_data["subplatform"].startswith("metadata") assert v1_data["availability_zone"] == client.instance.zone assert v1_data["instance_id"] == client.instance.instance_id diff --git a/tests/unittests/sources/test_init.py b/tests/unittests/sources/test_init.py index a266a148bf0..ce8fc9700cd 100644 --- a/tests/unittests/sources/test_init.py +++ b/tests/unittests/sources/test_init.py @@ -693,7 +693,7 @@ def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): "cloudinit.sources.canonical_cloud_id", return_value="my-cloud" ): datasource.get_data() - self.assertEqual("my-cloud", util.load_file(cloud_id_link)) + self.assertEqual("my-cloud\n", util.load_file(cloud_id_link)) # A symlink with the generic /run/cloud-init/cloud-id link is present self.assertTrue(util.is_link(cloud_id_link)) # When cloud-id changes, symlink and content change @@ -701,12 +701,12 @@ def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): "cloudinit.sources.canonical_cloud_id", return_value="my-cloud2" ): datasource.persist_instance_data() - self.assertEqual("my-cloud2", util.load_file(cloud_id2_file)) + self.assertEqual("my-cloud2\n", util.load_file(cloud_id2_file)) # Previous cloud-id- file removed self.assertFalse(os.path.exists(cloud_id_file)) # Generic link persisted which contains canonical-cloud-id as content self.assertTrue(util.is_link(cloud_id_link)) - self.assertEqual("my-cloud2", util.load_file(cloud_id_link)) + self.assertEqual("my-cloud2\n", util.load_file(cloud_id_link)) def test_persist_instance_data_writes_network_json_when_set(self): """When network_data.json class attribute is set, persist to json."""