Skip to content

sources/azure: remove lease file parsing #1302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TheRealFalcon merged 1 commit into from
Mar 8, 2022
Merged

sources/azure: remove lease file parsing #1302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions cloudinit/sources/DataSourceAzure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from cloudinit.sources.helpers.azure import (
DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE,
DEFAULT_WIRESERVER_ENDPOINT,
WALinuxAgentShim,
azure_ds_reporter,
azure_ds_telemetry_reporter,
build_minimal_ovf,
Expand All @@ -56,7 +57,6 @@
# azure systems will always have a resource disk, and 66-azure-ephemeral.rules
# ensures that it gets linked to this path.
RESOURCE_DISK_PATH = "/dev/disk/cloud/azure_resource"
LEASE_FILE = "/var/lib/dhcp/dhclient.eth0.leases"
DEFAULT_FS = "ext4"
# DMI chassis-asset-tag is set static for all azure instances
AZURE_CHASSIS_ASSET_TAG = "7783-7084-3265-9085-8269-3286-77"
Expand Down Expand Up @@ -271,7 +271,6 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]:

# update the FreeBSD specific information
if util.is_FreeBSD():
LEASE_FILE = "/var/db/dhclient.leases.hn0"
DEFAULT_FS = "freebsd-ufs"
res_disk = get_resource_disk_on_freebsd(1)
if res_disk is not None:
Expand All @@ -285,7 +284,6 @@ def get_resource_disk_on_freebsd(port_id) -> Optional[str]:
BUILTIN_DS_CONFIG = {
"data_dir": AGENT_SEED_DIR,
"disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH},
"dhclient_lease_file": LEASE_FILE,
"apply_network_config": True, # Use IMDS published network configuration
}
# RELEASE_BLOCKER: Xenial and earlier apply_network_config default is False
Expand Down Expand Up @@ -331,7 +329,6 @@ def __init__(self, sys_cfg, distro, paths):
self.ds_cfg = util.mergemanydict(
[util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG]
)
self.dhclient_lease_file = self.ds_cfg.get("dhclient_lease_file")
self._iso_dev = None
self._network_config = None
self._ephemeral_dhcp_ctx = None
Expand Down Expand Up @@ -432,7 +429,11 @@ def _setup_ephemeral_networking(

# Update wireserver IP from DHCP options.
if "unknown-245" in lease:
self._wireserver_endpoint = lease["unknown-245"]
self._wireserver_endpoint = (
WALinuxAgentShim.get_ip_from_lease_value(
lease["unknown-245"]
)
)

@azure_ds_telemetry_reporter
def _teardown_ephemeral_networking(self) -> None:
Expand Down Expand Up @@ -1362,7 +1363,7 @@ def _report_failure(self, description: Optional[str] = None) -> bool:
logger_func=LOG.debug,
)
report_failure_to_fabric(
dhcp_opts=self._wireserver_endpoint,
endpoint=self._wireserver_endpoint,
description=description,
)
return True
Expand All @@ -1385,7 +1386,7 @@ def _report_failure(self, description: Optional[str] = None) -> bool:
# Reporting failure will fail, but it will emit telemetry.
pass
report_failure_to_fabric(
dhcp_opts=self._wireserver_endpoint, description=description
endpoint=self._wireserver_endpoint, description=description
)
return True
except Exception as e:
Expand All @@ -1410,8 +1411,7 @@ def _report_ready(
"""
try:
data = get_metadata_from_fabric(
fallback_lease_file=None,
dhcp_opts=self._wireserver_endpoint,
endpoint=self._wireserver_endpoint,
iso_dev=self._iso_dev,
pubkey_info=pubkey_info,
)
Expand Down
218 changes: 20 additions & 198 deletions cloudinit/sources/helpers/azure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,14 @@
from xml.etree import ElementTree
from xml.sax.saxutils import escape

from cloudinit import (
distros,
stages,
subp,
temp_utils,
url_helper,
util,
version,
)
from cloudinit.net import dhcp
from cloudinit import distros, subp, temp_utils, url_helper, util, version
from cloudinit.reporting import events
from cloudinit.settings import CFG_BUILTIN

LOG = logging.getLogger(__name__)

# This endpoint matches the format as found in dhcp lease files, since this
# value is applied if the endpoint can't be found within a lease file
DEFAULT_WIRESERVER_ENDPOINT = "a8:3f:81:10"
# Default Wireserver endpoint (if not found in DHCP option 245).
DEFAULT_WIRESERVER_ENDPOINT = "168.63.129.16"

BOOT_EVENT_TYPE = "boot-telemetry"
SYSTEMINFO_EVENT_TYPE = "system-info"
Expand Down Expand Up @@ -325,14 +315,6 @@ def cd(newdir):
os.chdir(prevdir)


def _get_dhcp_endpoint_option_name():
if util.is_FreeBSD():
azure_endpoint = "option-245"
else:
azure_endpoint = "unknown-245"
return azure_endpoint


@azure_ds_telemetry_reporter
def http_with_retries(url, **kwargs) -> url_helper.UrlResponse:
"""Wrapper around url_helper.readurl() with custom telemetry logging
Expand Down Expand Up @@ -808,34 +790,15 @@ def _post_health_report(self, document: str) -> None:


class WALinuxAgentShim:
def __init__(self, fallback_lease_file=None, dhcp_options=None):
LOG.debug(
"WALinuxAgentShim instantiated, fallback_lease_file=%s",
fallback_lease_file,
)
self.dhcpoptions = dhcp_options
self._endpoint = None
self.openssl_manager = None
self.azure_endpoint_client = None
self.lease_file = fallback_lease_file
def __init__(self, endpoint: str):
self.endpoint = endpoint
self.openssl_manager: Optional[OpenSSLManager] = None
self.azure_endpoint_client: Optional[AzureEndpointHttpClient] = None

def clean_up(self):
if self.openssl_manager is not None:
self.openssl_manager.clean_up()

@staticmethod
def _get_hooks_dir():
_paths = stages.Init()
return os.path.join(_paths.paths.get_runpath(), "dhclient.hooks")

@property
def endpoint(self):
if self._endpoint is None:
self._endpoint = self.find_endpoint(
self.lease_file, self.dhcpoptions
)
return self._endpoint

@staticmethod
def get_ip_from_lease_value(fallback_lease_value):
unescaped_value = fallback_lease_value.replace("\\", "")
Expand All @@ -852,147 +815,6 @@ def get_ip_from_lease_value(fallback_lease_value):
packed_bytes = unescaped_value.encode("utf-8")
return socket.inet_ntoa(packed_bytes)

@staticmethod
@azure_ds_telemetry_reporter
def _networkd_get_value_from_leases(leases_d=None):
return dhcp.networkd_get_option_from_leases(
"OPTION_245", leases_d=leases_d
)

@staticmethod
@azure_ds_telemetry_reporter
def _get_value_from_leases_file(fallback_lease_file):
leases = []
try:
content = util.load_file(fallback_lease_file)
except IOError as ex:
LOG.error("Failed to read %s: %s", fallback_lease_file, ex)
return None

LOG.debug("content is %s", content)
option_name = _get_dhcp_endpoint_option_name()
for line in content.splitlines():
if option_name in line:
# Example line from Ubuntu
# option unknown-245 a8:3f:81:10;
leases.append(line.strip(" ").split(" ", 2)[-1].strip(';\n"'))
# Return the "most recent" one in the list
if len(leases) < 1:
return None
else:
return leases[-1]

@staticmethod
@azure_ds_telemetry_reporter
def _load_dhclient_json():
dhcp_options = {}
hooks_dir = WALinuxAgentShim._get_hooks_dir()
if not os.path.exists(hooks_dir):
LOG.debug("%s not found.", hooks_dir)
return None
hook_files = [
os.path.join(hooks_dir, x) for x in os.listdir(hooks_dir)
]
for hook_file in hook_files:
try:
name = os.path.basename(hook_file).replace(".json", "")
dhcp_options[name] = json.loads(util.load_file((hook_file)))
except ValueError as e:
raise ValueError(
"{_file} is not valid JSON data".format(_file=hook_file)
) from e
return dhcp_options

@staticmethod
@azure_ds_telemetry_reporter
def _get_value_from_dhcpoptions(dhcp_options):
if dhcp_options is None:
return None
# the MS endpoint server is given to us as DHPC option 245
_value = None
for interface in dhcp_options:
_value = dhcp_options[interface].get("unknown_245", None)
if _value is not None:
LOG.debug("Endpoint server found in dhclient options")
break
return _value

@staticmethod
@azure_ds_telemetry_reporter
def find_endpoint(fallback_lease_file=None, dhcp245=None):
"""Finds and returns the Azure endpoint using various methods.

The Azure endpoint is searched in the following order:
1. Endpoint from dhcp options (dhcp option 245).
2. Endpoint from networkd.
3. Endpoint from dhclient hook json.
4. Endpoint from fallback lease file.
5. The default Azure endpoint.

@param fallback_lease_file: Fallback lease file that will be used
during endpoint search.
@param dhcp245: dhcp options that will be used during endpoint search.
@return: Azure endpoint IP address.
"""
value = None

if dhcp245 is not None:
value = dhcp245
LOG.debug("Using Azure Endpoint from dhcp options")
if value is None:
report_diagnostic_event(
"No Azure endpoint from dhcp options. "
"Finding Azure endpoint from networkd...",
logger_func=LOG.debug,
)
value = WALinuxAgentShim._networkd_get_value_from_leases()
if value is None:
# Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
# a dhclient exit hook that calls cloud-init-dhclient-hook
report_diagnostic_event(
"No Azure endpoint from networkd. "
"Finding Azure endpoint from hook json...",
logger_func=LOG.debug,
)
dhcp_options = WALinuxAgentShim._load_dhclient_json()
value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
if value is None:
# Fallback and check the leases file if unsuccessful
report_diagnostic_event(
"No Azure endpoint from dhclient logs. "
"Unable to find endpoint in dhclient logs. "
"Falling back to check lease files",
logger_func=LOG.debug,
)
if fallback_lease_file is None:
report_diagnostic_event(
"No fallback lease file was specified.",
logger_func=LOG.warning,
)
value = None
else:
report_diagnostic_event(
"Looking for endpoint in lease file %s"
% fallback_lease_file,
logger_func=LOG.debug,
)
value = WALinuxAgentShim._get_value_from_leases_file(
fallback_lease_file
)
if value is None:
value = DEFAULT_WIRESERVER_ENDPOINT
report_diagnostic_event(
"No lease found; using default endpoint: %s" % value,
logger_func=LOG.warning,
)

endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value)
report_diagnostic_event(
"Azure endpoint found at %s" % endpoint_ip_address,
logger_func=LOG.debug,
)
return endpoint_ip_address

@azure_ds_telemetry_reporter
def eject_iso(self, iso_dev) -> None:
try:
Expand Down Expand Up @@ -1088,7 +910,7 @@ def _get_raw_goal_state_xml_from_azure(self) -> str:
description="retrieve goalstate",
parent=azure_ds_reporter,
):
response = self.azure_endpoint_client.get(url)
response = self.azure_endpoint_client.get(url) # type: ignore
except Exception as e:
report_diagnostic_event(
"failed to register with Azure and fetch GoalState XML: %s"
Expand All @@ -1112,7 +934,7 @@ def _parse_raw_goal_state_xml(
try:
goal_state = GoalState(
unparsed_goal_state_xml,
self.azure_endpoint_client,
self.azure_endpoint_client, # type: ignore
need_certificate,
)
except Exception as e:
Expand Down Expand Up @@ -1161,7 +983,11 @@ def _get_user_pubkeys(
@return: A list of the VM user's authorized pubkey values.
"""
ssh_keys = []
if goal_state.certificates_xml is not None and pubkey_info is not None:
if (
goal_state.certificates_xml is not None
and pubkey_info is not None
and self.openssl_manager is not None
):
LOG.debug("Certificate XML found; parsing out public keys.")
keys_by_fingerprint = self.openssl_manager.parse_certificates(
goal_state.certificates_xml
Expand Down Expand Up @@ -1208,11 +1034,11 @@ def _filter_pubkeys(keys_by_fingerprint: dict, pubkey_info: list) -> list:

@azure_ds_telemetry_reporter
def get_metadata_from_fabric(
fallback_lease_file=None, dhcp_opts=None, pubkey_info=None, iso_dev=None
endpoint: str,
pubkey_info: Optional[List[str]] = None,
iso_dev: Optional[str] = None,
):
shim = WALinuxAgentShim(
fallback_lease_file=fallback_lease_file, dhcp_options=dhcp_opts
)
shim = WALinuxAgentShim(endpoint=endpoint)
try:
return shim.register_with_azure_and_fetch_data(
pubkey_info=pubkey_info, iso_dev=iso_dev
Expand All @@ -1222,12 +1048,8 @@ def get_metadata_from_fabric(


@azure_ds_telemetry_reporter
def report_failure_to_fabric(
fallback_lease_file=None, dhcp_opts=None, description=None
):
shim = WALinuxAgentShim(
fallback_lease_file=fallback_lease_file, dhcp_options=dhcp_opts
)
def report_failure_to_fabric(endpoint: str, description: Optional[str] = None):
shim = WALinuxAgentShim(endpoint=endpoint)
if not description:
description = DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE
try:
Expand Down
Loading