diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 590c820e38b..ef5e07253d1 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -34,6 +34,7 @@ from cloudinit.sources.helpers.azure import ( DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE, DEFAULT_WIRESERVER_ENDPOINT, + WALinuxAgentShim, azure_ds_reporter, azure_ds_telemetry_reporter, build_minimal_ovf, @@ -56,7 +57,6 @@ # azure systems will always have a resource disk, and 66-azure-ephemeral.rules # ensures that it gets linked to this path. RESOURCE_DISK_PATH = "/dev/disk/cloud/azure_resource" -LEASE_FILE = "/var/lib/dhcp/dhclient.eth0.leases" DEFAULT_FS = "ext4" # DMI chassis-asset-tag is set static for all azure instances AZURE_CHASSIS_ASSET_TAG = "7783-7084-3265-9085-8269-3286-77" @@ -271,7 +271,6 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: # update the FreeBSD specific information if util.is_FreeBSD(): - LEASE_FILE = "/var/db/dhclient.leases.hn0" DEFAULT_FS = "freebsd-ufs" res_disk = get_resource_disk_on_freebsd(1) if res_disk is not None: @@ -285,7 +284,6 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]: BUILTIN_DS_CONFIG = { "data_dir": AGENT_SEED_DIR, "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, - "dhclient_lease_file": LEASE_FILE, "apply_network_config": True, # Use IMDS published network configuration } # RELEASE_BLOCKER: Xenial and earlier apply_network_config default is False @@ -331,7 +329,6 @@ def __init__(self, sys_cfg, distro, paths): self.ds_cfg = util.mergemanydict( [util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG] ) - self.dhclient_lease_file = self.ds_cfg.get("dhclient_lease_file") self._iso_dev = None self._network_config = None self._ephemeral_dhcp_ctx = None @@ -432,7 +429,11 @@ def _setup_ephemeral_networking( # Update wireserver IP from DHCP options. if "unknown-245" in lease: - self._wireserver_endpoint = lease["unknown-245"] + self._wireserver_endpoint = ( + WALinuxAgentShim.get_ip_from_lease_value( + lease["unknown-245"] + ) + ) @azure_ds_telemetry_reporter def _teardown_ephemeral_networking(self) -> None: @@ -1362,7 +1363,7 @@ def _report_failure(self, description: Optional[str] = None) -> bool: logger_func=LOG.debug, ) report_failure_to_fabric( - dhcp_opts=self._wireserver_endpoint, + endpoint=self._wireserver_endpoint, description=description, ) return True @@ -1385,7 +1386,7 @@ def _report_failure(self, description: Optional[str] = None) -> bool: # Reporting failure will fail, but it will emit telemetry. pass report_failure_to_fabric( - dhcp_opts=self._wireserver_endpoint, description=description + endpoint=self._wireserver_endpoint, description=description ) return True except Exception as e: @@ -1410,8 +1411,7 @@ def _report_ready( """ try: data = get_metadata_from_fabric( - fallback_lease_file=None, - dhcp_opts=self._wireserver_endpoint, + endpoint=self._wireserver_endpoint, iso_dev=self._iso_dev, pubkey_info=pubkey_info, ) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 0a9f1f459e2..a6edc816e14 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -16,24 +16,14 @@ from xml.etree import ElementTree from xml.sax.saxutils import escape -from cloudinit import ( - distros, - stages, - subp, - temp_utils, - url_helper, - util, - version, -) -from cloudinit.net import dhcp +from cloudinit import distros, subp, temp_utils, url_helper, util, version from cloudinit.reporting import events from cloudinit.settings import CFG_BUILTIN LOG = logging.getLogger(__name__) -# This endpoint matches the format as found in dhcp lease files, since this -# value is applied if the endpoint can't be found within a lease file -DEFAULT_WIRESERVER_ENDPOINT = "a8:3f:81:10" +# Default Wireserver endpoint (if not found in DHCP option 245). +DEFAULT_WIRESERVER_ENDPOINT = "168.63.129.16" BOOT_EVENT_TYPE = "boot-telemetry" SYSTEMINFO_EVENT_TYPE = "system-info" @@ -325,14 +315,6 @@ def cd(newdir): os.chdir(prevdir) -def _get_dhcp_endpoint_option_name(): - if util.is_FreeBSD(): - azure_endpoint = "option-245" - else: - azure_endpoint = "unknown-245" - return azure_endpoint - - @azure_ds_telemetry_reporter def http_with_retries(url, **kwargs) -> url_helper.UrlResponse: """Wrapper around url_helper.readurl() with custom telemetry logging @@ -808,34 +790,15 @@ def _post_health_report(self, document: str) -> None: class WALinuxAgentShim: - def __init__(self, fallback_lease_file=None, dhcp_options=None): - LOG.debug( - "WALinuxAgentShim instantiated, fallback_lease_file=%s", - fallback_lease_file, - ) - self.dhcpoptions = dhcp_options - self._endpoint = None - self.openssl_manager = None - self.azure_endpoint_client = None - self.lease_file = fallback_lease_file + def __init__(self, endpoint: str): + self.endpoint = endpoint + self.openssl_manager: Optional[OpenSSLManager] = None + self.azure_endpoint_client: Optional[AzureEndpointHttpClient] = None def clean_up(self): if self.openssl_manager is not None: self.openssl_manager.clean_up() - @staticmethod - def _get_hooks_dir(): - _paths = stages.Init() - return os.path.join(_paths.paths.get_runpath(), "dhclient.hooks") - - @property - def endpoint(self): - if self._endpoint is None: - self._endpoint = self.find_endpoint( - self.lease_file, self.dhcpoptions - ) - return self._endpoint - @staticmethod def get_ip_from_lease_value(fallback_lease_value): unescaped_value = fallback_lease_value.replace("\\", "") @@ -852,147 +815,6 @@ def get_ip_from_lease_value(fallback_lease_value): packed_bytes = unescaped_value.encode("utf-8") return socket.inet_ntoa(packed_bytes) - @staticmethod - @azure_ds_telemetry_reporter - def _networkd_get_value_from_leases(leases_d=None): - return dhcp.networkd_get_option_from_leases( - "OPTION_245", leases_d=leases_d - ) - - @staticmethod - @azure_ds_telemetry_reporter - def _get_value_from_leases_file(fallback_lease_file): - leases = [] - try: - content = util.load_file(fallback_lease_file) - except IOError as ex: - LOG.error("Failed to read %s: %s", fallback_lease_file, ex) - return None - - LOG.debug("content is %s", content) - option_name = _get_dhcp_endpoint_option_name() - for line in content.splitlines(): - if option_name in line: - # Example line from Ubuntu - # option unknown-245 a8:3f:81:10; - leases.append(line.strip(" ").split(" ", 2)[-1].strip(';\n"')) - # Return the "most recent" one in the list - if len(leases) < 1: - return None - else: - return leases[-1] - - @staticmethod - @azure_ds_telemetry_reporter - def _load_dhclient_json(): - dhcp_options = {} - hooks_dir = WALinuxAgentShim._get_hooks_dir() - if not os.path.exists(hooks_dir): - LOG.debug("%s not found.", hooks_dir) - return None - hook_files = [ - os.path.join(hooks_dir, x) for x in os.listdir(hooks_dir) - ] - for hook_file in hook_files: - try: - name = os.path.basename(hook_file).replace(".json", "") - dhcp_options[name] = json.loads(util.load_file((hook_file))) - except ValueError as e: - raise ValueError( - "{_file} is not valid JSON data".format(_file=hook_file) - ) from e - return dhcp_options - - @staticmethod - @azure_ds_telemetry_reporter - def _get_value_from_dhcpoptions(dhcp_options): - if dhcp_options is None: - return None - # the MS endpoint server is given to us as DHPC option 245 - _value = None - for interface in dhcp_options: - _value = dhcp_options[interface].get("unknown_245", None) - if _value is not None: - LOG.debug("Endpoint server found in dhclient options") - break - return _value - - @staticmethod - @azure_ds_telemetry_reporter - def find_endpoint(fallback_lease_file=None, dhcp245=None): - """Finds and returns the Azure endpoint using various methods. - - The Azure endpoint is searched in the following order: - 1. Endpoint from dhcp options (dhcp option 245). - 2. Endpoint from networkd. - 3. Endpoint from dhclient hook json. - 4. Endpoint from fallback lease file. - 5. The default Azure endpoint. - - @param fallback_lease_file: Fallback lease file that will be used - during endpoint search. - @param dhcp245: dhcp options that will be used during endpoint search. - @return: Azure endpoint IP address. - """ - value = None - - if dhcp245 is not None: - value = dhcp245 - LOG.debug("Using Azure Endpoint from dhcp options") - if value is None: - report_diagnostic_event( - "No Azure endpoint from dhcp options. " - "Finding Azure endpoint from networkd...", - logger_func=LOG.debug, - ) - value = WALinuxAgentShim._networkd_get_value_from_leases() - if value is None: - # Option-245 stored in /run/cloud-init/dhclient.hooks/.json - # a dhclient exit hook that calls cloud-init-dhclient-hook - report_diagnostic_event( - "No Azure endpoint from networkd. " - "Finding Azure endpoint from hook json...", - logger_func=LOG.debug, - ) - dhcp_options = WALinuxAgentShim._load_dhclient_json() - value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) - if value is None: - # Fallback and check the leases file if unsuccessful - report_diagnostic_event( - "No Azure endpoint from dhclient logs. " - "Unable to find endpoint in dhclient logs. " - "Falling back to check lease files", - logger_func=LOG.debug, - ) - if fallback_lease_file is None: - report_diagnostic_event( - "No fallback lease file was specified.", - logger_func=LOG.warning, - ) - value = None - else: - report_diagnostic_event( - "Looking for endpoint in lease file %s" - % fallback_lease_file, - logger_func=LOG.debug, - ) - value = WALinuxAgentShim._get_value_from_leases_file( - fallback_lease_file - ) - if value is None: - value = DEFAULT_WIRESERVER_ENDPOINT - report_diagnostic_event( - "No lease found; using default endpoint: %s" % value, - logger_func=LOG.warning, - ) - - endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) - report_diagnostic_event( - "Azure endpoint found at %s" % endpoint_ip_address, - logger_func=LOG.debug, - ) - return endpoint_ip_address - @azure_ds_telemetry_reporter def eject_iso(self, iso_dev) -> None: try: @@ -1088,7 +910,7 @@ def _get_raw_goal_state_xml_from_azure(self) -> str: description="retrieve goalstate", parent=azure_ds_reporter, ): - response = self.azure_endpoint_client.get(url) + response = self.azure_endpoint_client.get(url) # type: ignore except Exception as e: report_diagnostic_event( "failed to register with Azure and fetch GoalState XML: %s" @@ -1112,7 +934,7 @@ def _parse_raw_goal_state_xml( try: goal_state = GoalState( unparsed_goal_state_xml, - self.azure_endpoint_client, + self.azure_endpoint_client, # type: ignore need_certificate, ) except Exception as e: @@ -1161,7 +983,11 @@ def _get_user_pubkeys( @return: A list of the VM user's authorized pubkey values. """ ssh_keys = [] - if goal_state.certificates_xml is not None and pubkey_info is not None: + if ( + goal_state.certificates_xml is not None + and pubkey_info is not None + and self.openssl_manager is not None + ): LOG.debug("Certificate XML found; parsing out public keys.") keys_by_fingerprint = self.openssl_manager.parse_certificates( goal_state.certificates_xml @@ -1208,11 +1034,11 @@ def _filter_pubkeys(keys_by_fingerprint: dict, pubkey_info: list) -> list: @azure_ds_telemetry_reporter def get_metadata_from_fabric( - fallback_lease_file=None, dhcp_opts=None, pubkey_info=None, iso_dev=None + endpoint: str, + pubkey_info: Optional[List[str]] = None, + iso_dev: Optional[str] = None, ): - shim = WALinuxAgentShim( - fallback_lease_file=fallback_lease_file, dhcp_options=dhcp_opts - ) + shim = WALinuxAgentShim(endpoint=endpoint) try: return shim.register_with_azure_and_fetch_data( pubkey_info=pubkey_info, iso_dev=iso_dev @@ -1222,12 +1048,8 @@ def get_metadata_from_fabric( @azure_ds_telemetry_reporter -def report_failure_to_fabric( - fallback_lease_file=None, dhcp_opts=None, description=None -): - shim = WALinuxAgentShim( - fallback_lease_file=fallback_lease_file, dhcp_options=dhcp_opts - ) +def report_failure_to_fabric(endpoint: str, description: Optional[str] = None): + shim = WALinuxAgentShim(endpoint=endpoint) if not description: description = DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE try: diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst index 1bd03970e36..b73d7d38323 100644 --- a/doc/rtd/topics/datasources/azure.rst +++ b/doc/rtd/topics/datasources/azure.rst @@ -11,20 +11,6 @@ CD formatted in UDF. That CD contains a 'ovf-env.xml' file that provides some information. Additional information is obtained via interaction with the "endpoint". -To find the endpoint, we now leverage the dhcp client's ability to log its -known values on exit. The endpoint server is special DHCP option 245. -Depending on your networking stack, this can be done -by calling a script in /etc/dhcp/dhclient-exit-hooks or a file in -/etc/NetworkManager/dispatcher.d. Both of these call a sub-command -'dhclient_hook' of cloud-init itself. This sub-command will write the client -information in json format to /run/cloud-init/dhclient.hook/.json. - -If those files are not available, the fallback is to check the leases file -for the endpoint server (again option 245). - -You can define the path to the lease file with the 'dhclient_lease_file' -configuration. - IMDS ---- @@ -56,8 +42,6 @@ The settings that may be configured are: dhcp on eth0. Default is True. For Ubuntu 16.04 or earlier, default is False. * **data_dir**: Path used to read metadata files and write crawled data. - * **dhclient_lease_file**: The fallback lease file to source when looking for - custom DHCP option 245 from Azure fabric. * **disk_aliases**: A dictionary defining which device paths should be interpreted as ephemeral images. See cc_disk_setup module for more info. @@ -74,7 +58,6 @@ An example configuration with the default values is provided below: Azure: apply_network_config: true data_dir: /var/lib/waagent - dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases disk_aliases: ephemeral0: /dev/disk/cloud/azure_resource diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index 2eb0fa8caf9..62e5134b709 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -114,7 +114,7 @@ def mock_net_dhcp_maybe_perform_dhcp_discovery(): "cloudinit.net.dhcp.maybe_perform_dhcp_discovery", return_value=[ { - "unknown-245": "aa:bb:cc:dd", + "unknown-245": "0a:0b:0c:0d", "interface": "ethBoot0", "fixed-address": "192.168.2.9", "routers": "192.168.2.1", @@ -1887,7 +1887,7 @@ def test_dsaz_report_failure_description_msg(self): test_msg = "Test report failure description message" self.assertTrue(dsrc._report_failure(description=test_msg)) self.m_report_failure_to_fabric.assert_called_once_with( - dhcp_opts=mock.ANY, description=test_msg + endpoint="168.63.129.16", description=test_msg ) def test_dsaz_report_failure_no_description_msg(self): @@ -1898,7 +1898,7 @@ def test_dsaz_report_failure_no_description_msg(self): self.assertTrue(dsrc._report_failure()) # no description msg self.m_report_failure_to_fabric.assert_called_once_with( - dhcp_opts=mock.ANY, description=None + endpoint="168.63.129.16", description=None ) def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): @@ -1907,8 +1907,8 @@ def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): with mock.patch.object( dsrc, "crawl_metadata" ) as m_crawl_metadata, mock.patch.object( - dsrc, "_wireserver_endpoint", return_value="test-ep" - ) as m_wireserver_endpoint: + dsrc, "_wireserver_endpoint", "test-ep" + ): # mock crawl metadata failure to cause report failure m_crawl_metadata.side_effect = Exception @@ -1916,7 +1916,7 @@ def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): # ensure called with cached ephemeral dhcp lease option 245 self.m_report_failure_to_fabric.assert_called_once_with( - description=mock.ANY, dhcp_opts=m_wireserver_endpoint + endpoint="test-ep", description=mock.ANY ) def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): @@ -1926,7 +1926,7 @@ def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): # mock crawl metadata failure to cause report failure m_crawl_metadata.side_effect = Exception - test_lease_dhcp_option_245 = "test_lease_dhcp_option_245" + test_lease_dhcp_option_245 = "01:02:03:04" test_lease = { "unknown-245": test_lease_dhcp_option_245, "interface": "eth0", @@ -1938,7 +1938,7 @@ def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): # ensure called with the newly discovered # ephemeral dhcp lease option 245 self.m_report_failure_to_fabric.assert_called_once_with( - description=mock.ANY, dhcp_opts=test_lease_dhcp_option_245 + endpoint="1.2.3.4", description=mock.ANY ) def test_exception_fetching_fabric_data_doesnt_propagate(self): @@ -3671,24 +3671,17 @@ def test_non_ascii_seed_is_serializable(self): class TestEphemeralNetworking: @pytest.mark.parametrize("iface", [None, "fakeEth0"]) - @pytest.mark.parametrize( - "lease", - [ - { - "interface": "fakeEth0", - "unknown-245": "00:11:22:33", - }, - {"interface": "fakeEth0"}, - ], - ) def test_basic_setup( self, azure_ds, mock_ephemeral_dhcp_v4, mock_sleep, iface, - lease, ): + lease = { + "interface": "fakeEth0", + "unknown-245": "10:ff:fe:fd", + } mock_ephemeral_dhcp_v4.return_value.obtain_lease.side_effect = [lease] azure_ds._setup_ephemeral_networking(iface=iface) @@ -3698,9 +3691,30 @@ def test_basic_setup( mock.call().obtain_lease(), ] assert mock_sleep.mock_calls == [] - assert azure_ds._wireserver_endpoint == lease.get( - "unknown-245", "a8:3f:81:10" - ) + assert azure_ds._wireserver_endpoint == "16.255.254.253" + assert azure_ds._ephemeral_dhcp_ctx.iface == lease["interface"] + + @pytest.mark.parametrize("iface", [None, "fakeEth0"]) + def test_basic_setup_without_wireserver_opt( + self, + azure_ds, + mock_ephemeral_dhcp_v4, + mock_sleep, + iface, + ): + lease = { + "interface": "fakeEth0", + } + mock_ephemeral_dhcp_v4.return_value.obtain_lease.side_effect = [lease] + + azure_ds._setup_ephemeral_networking(iface=iface) + + assert mock_ephemeral_dhcp_v4.mock_calls == [ + mock.call(iface=iface, dhcp_log_func=dsaz.dhcp_log_cb), + mock.call().obtain_lease(), + ] + assert mock_sleep.mock_calls == [] + assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx.iface == lease["interface"] def test_no_retry_missing_dhclient_error( @@ -3740,7 +3754,7 @@ def test_retry_interface_error( mock.call().obtain_lease(), ] assert mock_sleep.mock_calls == [mock.call(1)] - assert azure_ds._wireserver_endpoint == "a8:3f:81:10" + assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx.iface == "fakeEth0" @pytest.mark.parametrize( @@ -3770,7 +3784,7 @@ def test_retry_sleeps( + [mock.call().obtain_lease()] * 11 ) assert mock_sleep.mock_calls == [mock.call(1)] * 10 - assert azure_ds._wireserver_endpoint == "a8:3f:81:10" + assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx.iface == "fakeEth0" @pytest.mark.parametrize( @@ -3806,7 +3820,7 @@ def test_retry_times_out( == [mock.call.obtain_lease()] * 3 ) assert mock_sleep.mock_calls == [mock.call(1)] * 2 - assert azure_ds._wireserver_endpoint == "a8:3f:81:10" + assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx is None @@ -4105,7 +4119,7 @@ def test_no_pps(self): assert self.mock_net_dhcp_maybe_perform_dhcp_discovery.mock_calls == [ mock.call(None, dsaz.dhcp_log_cb) ] - assert self.azure_ds._wireserver_endpoint == "aa:bb:cc:dd" + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" assert self.azure_ds._is_ephemeral_networking_up() is False # Verify DMI usage. @@ -4120,8 +4134,7 @@ def test_no_pps(self): # Verify reporting ready once. assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ mock.call( - fallback_lease_file=None, - dhcp_opts="aa:bb:cc:dd", + endpoint="10.11.12.13", iso_dev="/dev/sr0", pubkey_info=None, ) @@ -4194,7 +4207,7 @@ def test_running_pps(self): mock.call(None, dsaz.dhcp_log_cb), mock.call(None, dsaz.dhcp_log_cb), ] - assert self.azure_ds._wireserver_endpoint == "aa:bb:cc:dd" + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" assert self.azure_ds._is_ephemeral_networking_up() is False # Verify DMI usage. @@ -4209,14 +4222,12 @@ def test_running_pps(self): # Verify reporting ready twice. assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ mock.call( - fallback_lease_file=None, - dhcp_opts="aa:bb:cc:dd", + endpoint="10.11.12.13", iso_dev="/dev/sr0", pubkey_info=None, ), mock.call( - fallback_lease_file=None, - dhcp_opts="aa:bb:cc:dd", + endpoint="10.11.12.13", iso_dev=None, pubkey_info=None, ), @@ -4315,7 +4326,7 @@ def test_savable_pps(self): mock.call(None, dsaz.dhcp_log_cb), mock.call("ethAttached1", dsaz.dhcp_log_cb), ] - assert self.azure_ds._wireserver_endpoint == "aa:bb:cc:dd" + assert self.azure_ds._wireserver_endpoint == "10.11.12.13" assert self.azure_ds._is_ephemeral_networking_up() is False # Verify DMI usage. @@ -4330,14 +4341,12 @@ def test_savable_pps(self): # Verify reporting ready twice. assert self.mock_azure_get_metadata_from_fabric.mock_calls == [ mock.call( - fallback_lease_file=None, - dhcp_opts="aa:bb:cc:dd", + endpoint="10.11.12.13", iso_dev="/dev/sr0", pubkey_info=None, ), mock.call( - fallback_lease_file=None, - dhcp_opts="aa:bb:cc:dd", + endpoint="10.11.12.13", iso_dev=None, pubkey_info=None, ), diff --git a/tests/unittests/sources/test_azure_helper.py b/tests/unittests/sources/test_azure_helper.py index 98143bc33b9..bcdccd831c9 100644 --- a/tests/unittests/sources/test_azure_helper.py +++ b/tests/unittests/sources/test_azure_helper.py @@ -11,7 +11,7 @@ from cloudinit.sources.helpers import azure as azure_helper from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim from cloudinit.util import load_file -from tests.unittests.helpers import CiTestCase, ExitStack, mock, populate_dir +from tests.unittests.helpers import CiTestCase, ExitStack, mock GOAL_STATE_TEMPLATE = """\ @@ -87,70 +87,6 @@ class SentinelException(Exception): pass -class TestFindEndpoint(CiTestCase): - def setUp(self): - super(TestFindEndpoint, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.load_file = patches.enter_context( - mock.patch.object(azure_helper.util, "load_file") - ) - - self.dhcp_options = patches.enter_context( - mock.patch.object(wa_shim, "_load_dhclient_json") - ) - - self.networkd_leases = patches.enter_context( - mock.patch.object(wa_shim, "_networkd_get_value_from_leases") - ) - self.networkd_leases.return_value = None - - def test_missing_file(self): - """wa_shim find_endpoint uses default endpoint if - leasefile not found - """ - self.assertEqual(wa_shim.find_endpoint(), "168.63.129.16") - - def test_missing_special_azure_line(self): - """wa_shim find_endpoint uses default endpoint if leasefile is found - but does not contain DHCP Option 245 (whose value is the endpoint) - """ - self.load_file.return_value = "" - self.dhcp_options.return_value = {"eth0": {"key": "value"}} - self.assertEqual(wa_shim.find_endpoint(), "168.63.129.16") - - @staticmethod - def _build_lease_content(encoded_address): - endpoint = azure_helper._get_dhcp_endpoint_option_name() - return "\n".join( - [ - "lease {", - ' interface "eth0";', - " option {0} {1};".format(endpoint, encoded_address), - "}", - ] - ) - - def test_from_dhcp_client(self): - self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}} - self.assertEqual("5.4.3.2", wa_shim.find_endpoint(None)) - - def test_latest_lease_used(self): - encoded_addresses = ["5:4:3:2", "4:3:2:1"] - file_content = "\n".join( - [ - self._build_lease_content(encoded_address) - for encoded_address in encoded_addresses - ] - ) - self.load_file.return_value = file_content - self.assertEqual( - encoded_addresses[-1].replace(":", "."), - wa_shim.find_endpoint("foobar"), - ) - - class TestExtractIpAddressFromLeaseValue(CiTestCase): def test_hex_string(self): ip_address, encoded_address = "98.76.54.32", "62:4c:36:20" @@ -1101,9 +1037,6 @@ def setUp(self): self.AzureEndpointHttpClient = patches.enter_context( mock.patch.object(azure_helper, "AzureEndpointHttpClient") ) - self.find_endpoint = patches.enter_context( - mock.patch.object(wa_shim, "find_endpoint") - ) self.GoalState = patches.enter_context( mock.patch.object(azure_helper, "GoalState") ) @@ -1122,7 +1055,7 @@ def setUp(self): self.GoalState.return_value.instance_id = self.test_instance_id def test_eject_iso_is_called(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") with mock.patch.object( shim, "eject_iso", autospec=True ) as m_eject_iso: @@ -1130,22 +1063,21 @@ def test_eject_iso_is_called(self): m_eject_iso.assert_called_once_with("/dev/sr0") def test_http_client_does_not_use_certificate_for_report_ready(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(None)], self.AzureEndpointHttpClient.call_args_list ) def test_http_client_does_not_use_certificate_for_report_failure(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") self.assertEqual( [mock.call(None)], self.AzureEndpointHttpClient.call_args_list ) def test_correct_url_used_for_goalstate_during_report_ready(self): - self.find_endpoint.return_value = "test_endpoint" - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() m_get = self.AzureEndpointHttpClient.return_value.get self.assertEqual( @@ -1164,8 +1096,7 @@ def test_correct_url_used_for_goalstate_during_report_ready(self): ) def test_correct_url_used_for_goalstate_during_report_failure(self): - self.find_endpoint.return_value = "test_endpoint" - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") m_get = self.AzureEndpointHttpClient.return_value.get self.assertEqual( @@ -1187,7 +1118,7 @@ def test_certificates_used_to_determine_public_keys(self): # if register_with_azure_and_fetch_data() isn't passed some info about # the user's public keys, there's no point in even trying to parse the # certificates - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") mypk = [ {"fingerprint": "fp1", "path": "path1"}, {"fingerprint": "fp3", "path": "path3", "value": ""}, @@ -1211,13 +1142,12 @@ def test_certificates_used_to_determine_public_keys(self): def test_absent_certificates_produces_empty_public_keys(self): mypk = [{"fingerprint": "fp1", "path": "path1"}] self.GoalState.return_value.certificates_xml = None - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") data = shim.register_with_azure_and_fetch_data(pubkey_info=mypk) self.assertEqual([], data) def test_correct_url_used_for_report_ready(self): - self.find_endpoint.return_value = "test_endpoint" - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() expected_url = "http://test_endpoint/machine?comp=health" self.assertEqual( @@ -1226,8 +1156,7 @@ def test_correct_url_used_for_report_ready(self): ) def test_correct_url_used_for_report_failure(self): - self.find_endpoint.return_value = "test_endpoint" - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") expected_url = "http://test_endpoint/machine?comp=health" self.assertEqual( @@ -1236,7 +1165,7 @@ def test_correct_url_used_for_report_failure(self): ) def test_goal_state_values_used_for_report_ready(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]["data"] @@ -1246,7 +1175,7 @@ def test_goal_state_values_used_for_report_ready(self): self.assertIn(self.test_instance_id, posted_document) def test_goal_state_values_used_for_report_failure(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]["data"] @@ -1256,7 +1185,7 @@ def test_goal_state_values_used_for_report_failure(self): self.assertIn(self.test_instance_id, posted_document) def test_xml_elems_in_report_ready_post(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() health_document = HEALTH_REPORT_XML_TEMPLATE.format( incarnation=escape(self.test_incarnation), @@ -1271,7 +1200,7 @@ def test_xml_elems_in_report_ready_post(self): self.assertEqual(health_document, posted_document) def test_xml_elems_in_report_failure_post(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") health_document = HEALTH_REPORT_XML_TEMPLATE.format( incarnation=escape(self.test_incarnation), @@ -1294,7 +1223,7 @@ def test_xml_elems_in_report_failure_post(self): def test_register_with_azure_and_fetch_data_calls_send_ready_signal( self, m_goal_state_health_reporter ): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() self.assertEqual( 1, @@ -1305,7 +1234,7 @@ def test_register_with_azure_and_fetch_data_calls_send_ready_signal( def test_register_with_azure_and_report_failure_calls_send_failure_signal( self, m_goal_state_health_reporter ): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") m_goal_state_health_reporter.return_value.send_failure_signal.assert_called_once_with( # noqa: E501 description="TestDesc" @@ -1314,7 +1243,7 @@ def test_register_with_azure_and_report_failure_calls_send_failure_signal( def test_register_with_azure_and_report_failure_does_not_need_certificates( self, ): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") with mock.patch.object( shim, "_fetch_goal_state_from_azure", autospec=True ) as m_fetch_goal_state_from_azure: @@ -1324,24 +1253,24 @@ def test_register_with_azure_and_report_failure_does_not_need_certificates( ) def test_clean_up_can_be_called_at_any_time(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.clean_up() def test_openssl_manager_not_instantiated_by_shim_report_status(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() shim.register_with_azure_and_report_failure(description="TestDesc") shim.clean_up() self.OpenSSLManager.assert_not_called() def test_clean_up_after_report_ready(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data() shim.clean_up() self.OpenSSLManager.return_value.clean_up.assert_not_called() def test_clean_up_after_report_failure(self): - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") shim.clean_up() self.OpenSSLManager.return_value.clean_up.assert_not_called() @@ -1350,7 +1279,7 @@ def test_fetch_goalstate_during_report_ready_raises_exc_on_get_exc(self): self.AzureEndpointHttpClient.return_value.get.side_effect = ( SentinelException ) - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") self.assertRaises( SentinelException, shim.register_with_azure_and_fetch_data ) @@ -1359,7 +1288,7 @@ def test_fetch_goalstate_during_report_failure_raises_exc_on_get_exc(self): self.AzureEndpointHttpClient.return_value.get.side_effect = ( SentinelException ) - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") self.assertRaises( SentinelException, shim.register_with_azure_and_report_failure, @@ -1368,7 +1297,7 @@ def test_fetch_goalstate_during_report_failure_raises_exc_on_get_exc(self): def test_fetch_goalstate_during_report_ready_raises_exc_on_parse_exc(self): self.GoalState.side_effect = SentinelException - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") self.assertRaises( SentinelException, shim.register_with_azure_and_fetch_data ) @@ -1377,7 +1306,7 @@ def test_fetch_goalstate_during_report_failure_raises_exc_on_parse_exc( self, ): self.GoalState.side_effect = SentinelException - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") self.assertRaises( SentinelException, shim.register_with_azure_and_report_failure, @@ -1388,7 +1317,7 @@ def test_failure_to_send_report_ready_health_doc_bubbles_up(self): self.AzureEndpointHttpClient.return_value.post.side_effect = ( SentinelException ) - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") self.assertRaises( SentinelException, shim.register_with_azure_and_fetch_data ) @@ -1397,7 +1326,7 @@ def test_failure_to_send_report_failure_health_doc_bubbles_up(self): self.AzureEndpointHttpClient.return_value.post.side_effect = ( SentinelException ) - shim = wa_shim() + shim = wa_shim(endpoint="test_endpoint") self.assertRaises( SentinelException, shim.register_with_azure_and_report_failure, @@ -1416,14 +1345,14 @@ def setUp(self): ) def test_data_from_shim_returned(self): - ret = azure_helper.get_metadata_from_fabric() + ret = azure_helper.get_metadata_from_fabric(endpoint="test_endpoint") self.assertEqual( self.m_shim.return_value.register_with_azure_and_fetch_data.return_value, # noqa: E501 ret, ) def test_success_calls_clean_up(self): - azure_helper.get_metadata_from_fabric() + azure_helper.get_metadata_from_fabric(endpoint="test_endpoint") self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) def test_failure_in_registration_propagates_exc_and_calls_clean_up(self): @@ -1431,14 +1360,18 @@ def test_failure_in_registration_propagates_exc_and_calls_clean_up(self): SentinelException ) self.assertRaises( - SentinelException, azure_helper.get_metadata_from_fabric + SentinelException, + azure_helper.get_metadata_from_fabric, + "test_endpoint", ) self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) def test_calls_shim_register_with_azure_and_fetch_data(self): m_pubkey_info = mock.MagicMock() azure_helper.get_metadata_from_fabric( - pubkey_info=m_pubkey_info, iso_dev="/dev/sr0" + endpoint="test_endpoint", + pubkey_info=m_pubkey_info, + iso_dev="/dev/sr0", ) self.assertEqual( 1, @@ -1450,17 +1383,10 @@ def test_calls_shim_register_with_azure_and_fetch_data(self): ) def test_instantiates_shim_with_kwargs(self): - m_fallback_lease_file = mock.MagicMock() - m_dhcp_options = mock.MagicMock() - azure_helper.get_metadata_from_fabric( - fallback_lease_file=m_fallback_lease_file, dhcp_opts=m_dhcp_options - ) + azure_helper.get_metadata_from_fabric(endpoint="test_endpoint") self.assertEqual(1, self.m_shim.call_count) self.assertEqual( - mock.call( - fallback_lease_file=m_fallback_lease_file, - dhcp_options=m_dhcp_options, - ), + mock.call(endpoint="test_endpoint"), self.m_shim.call_args, ) @@ -1478,7 +1404,7 @@ def setUp(self): ) def test_success_calls_clean_up(self): - azure_helper.report_failure_to_fabric() + azure_helper.report_failure_to_fabric(endpoint="test_endpoint") self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) def test_failure_in_shim_report_failure_propagates_exc_and_calls_clean_up( @@ -1488,14 +1414,18 @@ def test_failure_in_shim_report_failure_propagates_exc_and_calls_clean_up( SentinelException ) self.assertRaises( - SentinelException, azure_helper.report_failure_to_fabric + SentinelException, + azure_helper.report_failure_to_fabric, + "test_endpoint", ) self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) def test_report_failure_to_fabric_with_desc_calls_shim_report_failure( self, ): - azure_helper.report_failure_to_fabric(description="TestDesc") + azure_helper.report_failure_to_fabric( + endpoint="test_endpoint", description="TestDesc" + ) self.m_shim.return_value.register_with_azure_and_report_failure.assert_called_once_with( # noqa: E501 description="TestDesc" ) @@ -1503,7 +1433,7 @@ def test_report_failure_to_fabric_with_desc_calls_shim_report_failure( def test_report_failure_to_fabric_with_no_desc_calls_shim_report_failure( self, ): - azure_helper.report_failure_to_fabric() + azure_helper.report_failure_to_fabric(endpoint="test_endpoint") # default err message description should be shown to the user # if no description is passed in self.m_shim.return_value.register_with_azure_and_report_failure.assert_called_once_with( # noqa: E501 @@ -1515,7 +1445,9 @@ def test_report_failure_to_fabric_with_no_desc_calls_shim_report_failure( def test_report_failure_to_fabric_empty_desc_calls_shim_report_failure( self, ): - azure_helper.report_failure_to_fabric(description="") + azure_helper.report_failure_to_fabric( + endpoint="test_endpoint", description="" + ) # default err message description should be shown to the user # if an empty description is passed in self.m_shim.return_value.register_with_azure_and_report_failure.assert_called_once_with( # noqa: E501 @@ -1525,84 +1457,11 @@ def test_report_failure_to_fabric_empty_desc_calls_shim_report_failure( ) def test_instantiates_shim_with_kwargs(self): - m_fallback_lease_file = mock.MagicMock() - m_dhcp_options = mock.MagicMock() azure_helper.report_failure_to_fabric( - fallback_lease_file=m_fallback_lease_file, dhcp_opts=m_dhcp_options + endpoint="test_endpoint", ) self.m_shim.assert_called_once_with( - fallback_lease_file=m_fallback_lease_file, - dhcp_options=m_dhcp_options, - ) - - -class TestExtractIpAddressFromNetworkd(CiTestCase): - - azure_lease = dedent( - """\ - # This is private data. Do not parse. - ADDRESS=10.132.0.5 - NETMASK=255.255.255.255 - ROUTER=10.132.0.1 - SERVER_ADDRESS=169.254.169.254 - NEXT_SERVER=10.132.0.1 - MTU=1460 - T1=43200 - T2=75600 - LIFETIME=86400 - DNS=169.254.169.254 - NTP=169.254.169.254 - DOMAINNAME=c.ubuntu-foundations.internal - DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal - HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal - ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 - CLIENTID=ff405663a200020000ab11332859494d7a8b4c - OPTION_245=624c3620 - """ - ) - - def setUp(self): - super(TestExtractIpAddressFromNetworkd, self).setUp() - self.lease_d = self.tmp_dir() - - def test_no_valid_leases_is_none(self): - """No valid leases should return None.""" - self.assertIsNone( - wa_shim._networkd_get_value_from_leases(self.lease_d) - ) - - def test_option_245_is_found_in_single(self): - """A single valid lease with 245 option should return it.""" - populate_dir(self.lease_d, {"9": self.azure_lease}) - self.assertEqual( - "624c3620", wa_shim._networkd_get_value_from_leases(self.lease_d) - ) - - def test_option_245_not_found_returns_None(self): - """A valid lease, but no option 245 should return None.""" - populate_dir( - self.lease_d, - {"9": self.azure_lease.replace("OPTION_245", "OPTION_999")}, - ) - self.assertIsNone( - wa_shim._networkd_get_value_from_leases(self.lease_d) - ) - - def test_multiple_returns_first(self): - """Somewhat arbitrarily return the first address when multiple. - - Most important at the moment is that this is consistent behavior - rather than changing randomly as in order of a dictionary.""" - myval = "624c3601" - populate_dir( - self.lease_d, - { - "9": self.azure_lease, - "2": self.azure_lease.replace("624c3620", myval), - }, - ) - self.assertEqual( - myval, wa_shim._networkd_get_value_from_leases(self.lease_d) + endpoint="test_endpoint", )