diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py index 7e18fc17ef8..ff25b3fa074 100644 --- a/cloudinit/sources/DataSourceVMware.py +++ b/cloudinit/sources/DataSourceVMware.py @@ -16,51 +16,6 @@ * EnvVars * GuestInfo * IMC (Guest Customization) - -Netifaces (https://github.com/al45tair/netifaces) - - Please note this module relies on the netifaces project to introspect the - runtime, network configuration of the host on which this datasource is - running. This is in contrast to the rest of cloud-init which uses the - cloudinit/netinfo module. - - The reasons for using netifaces include: - - * Netifaces is built in C and is more portable across multiple systems - and more deterministic than shell exec'ing local network commands and - parsing their output. - - * Netifaces provides a stable way to determine the view of the host's - network after DHCP has brought the network online. Unlike most other - datasources, this datasource still provides support for JINJA queries - based on networking information even when the network is based on a - DHCP lease. While this does not tie this datasource directly to - netifaces, it does mean the ability to consistently obtain the - correct information is paramount. - - * It is currently possible to execute this datasource on macOS - (which many developers use today) to print the output of the - get_host_info function. This function calls netifaces to obtain - the same runtime network configuration that the datasource would - persist to the local system's instance data. - - However, the netinfo module fails on macOS. The result is either a - hung operation that requires a SIGINT to return control to the user, - or, if brew is used to install iproute2mac, the ip commands are used - but produce output the netinfo module is unable to parse. - - While macOS is not a target of cloud-init, this feature is quite - useful when working on this datasource. - - For more information about this behavior, please see the following - PR comment, https://bit.ly/3fG7OVh. - - The authors of this datasource are not opposed to moving away from - netifaces. The goal may be to eventually do just that. This proviso was - added to the top of this module as a way to remind future-us and others - why netifaces was used in the first place in order to either smooth the - transition away from netifaces or embrace it further up the cloud-init - stack. """ import collections @@ -72,9 +27,7 @@ import socket import time -import netifaces - -from cloudinit import atomic_helper, dmi, log, net, sources, util +from cloudinit import atomic_helper, dmi, log, net, netinfo, sources, util from cloudinit.sources.helpers.vmware.imc import guestcust_util from cloudinit.subp import ProcessExecutionError, subp, which @@ -776,91 +729,64 @@ def get_default_ip_addrs(): addresses associated with the device used by the default route for a given address. """ - # TODO(promote and use netifaces in cloudinit.net* modules) - gateways = netifaces.gateways() - if "default" not in gateways: - return None, None - - default_gw = gateways["default"] - if ( - netifaces.AF_INET not in default_gw - and netifaces.AF_INET6 not in default_gw - ): - return None, None + # Get ipv4 and ipv6 interfaces associated with default routes + ipv4_if = None + ipv6_if = None + routes = netinfo.route_info() + for route in routes["ipv4"]: + if route["destination"] == "0.0.0.0": + ipv4_if = route["iface"] + break + for route in routes["ipv6"]: + if route["destination"] == "::/0": + ipv6_if = route["iface"] + break + + # Get ip address associated with default interface ipv4 = None ipv6 = None - - gw4 = default_gw.get(netifaces.AF_INET) - if gw4: - _, dev4 = gw4 - addr4_fams = netifaces.ifaddresses(dev4) - if addr4_fams: - af_inet4 = addr4_fams.get(netifaces.AF_INET) - if af_inet4: - if len(af_inet4) > 1: - LOG.debug( - "device %s has more than one ipv4 address: %s", - dev4, - af_inet4, - ) - elif "addr" in af_inet4[0]: - ipv4 = af_inet4[0]["addr"] - - # Try to get the default IPv6 address by first seeing if there is a default - # IPv6 route. - gw6 = default_gw.get(netifaces.AF_INET6) - if gw6: - _, dev6 = gw6 - addr6_fams = netifaces.ifaddresses(dev6) - if addr6_fams: - af_inet6 = addr6_fams.get(netifaces.AF_INET6) - if af_inet6: - if len(af_inet6) > 1: - LOG.debug( - "device %s has more than one ipv6 address: %s", - dev6, - af_inet6, - ) - elif "addr" in af_inet6[0]: - ipv6 = af_inet6[0]["addr"] + netdev = netinfo.netdev_info() + if ipv4_if in netdev: + addrs = netdev[ipv4_if]["ipv4"] + if len(addrs) > 1: + LOG.debug( + "device %s has more than one ipv4 address: %s", ipv4_if, addrs + ) + elif len(addrs) == 1 and "ip" in addrs[0]: + ipv4 = addrs[0]["ip"] + if ipv6_if in netdev: + addrs = netdev[ipv6_if]["ipv6"] + if len(addrs) > 1: + LOG.debug( + "device %s has more than one ipv6 address: %s", ipv6_if, addrs + ) + elif len(addrs) == 1 and "ip" in addrs[0]: + ipv6 = addrs[0]["ip"] # If there is a default IPv4 address but not IPv6, then see if there is a # single IPv6 address associated with the same device associated with the # default IPv4 address. - if ipv4 and not ipv6: - af_inet6 = addr4_fams.get(netifaces.AF_INET6) - if af_inet6: - if len(af_inet6) > 1: - LOG.debug( - "device %s has more than one ipv6 address: %s", - dev4, - af_inet6, - ) - elif "addr" in af_inet6[0]: - ipv6 = af_inet6[0]["addr"] + if ipv4 is not None and ipv6 is None: + for dev_name in netdev: + for addr in netdev[dev_name]["ipv4"]: + if addr["ip"] == ipv4 and len(netdev[dev_name]["ipv6"]) == 1: + ipv6 = netdev[dev_name]["ipv6"][0]["ip"] + break # If there is a default IPv6 address but not IPv4, then see if there is a # single IPv4 address associated with the same device associated with the # default IPv6 address. - if not ipv4 and ipv6: - af_inet4 = addr6_fams.get(netifaces.AF_INET) - if af_inet4: - if len(af_inet4) > 1: - LOG.debug( - "device %s has more than one ipv4 address: %s", - dev6, - af_inet4, - ) - elif "addr" in af_inet4[0]: - ipv4 = af_inet4[0]["addr"] + if ipv4 is None and ipv6 is not None: + for dev_name in netdev: + for addr in netdev[dev_name]["ipv6"]: + if addr["ip"] == ipv6 and len(netdev[dev_name]["ipv4"]) == 1: + ipv4 = netdev[dev_name]["ipv4"][0]["ip"] + break return ipv4, ipv6 -# patched socket.getfqdn() - see https://bugs.python.org/issue5004 - - def getfqdn(name=""): """Get fully qualified domain name from name. An empty argument is interpreted as meaning the local host. @@ -895,6 +821,33 @@ def is_valid_ip_addr(val): ) +def convert_to_netifaces_format(addr): + """ + Takes a cloudinit.netinfo formatted address and converts to netifaces + format, since this module was originally written with netifaces as the + network introspection module. + netifaces format: + { + "broadcast": "10.15.255.255", + "netmask": "255.240.0.0", + "addr": "10.0.1.4" + } + + cloudinit.netinfo format: + { + "ip": "10.0.1.4", + "mask": "255.240.0.0", + "bcast": "10.15.255.255", + "scope": "global", + } + """ + return { + "broadcast": addr["bcast"], + "netmask": addr["mask"], + "addr": addr["ip"], + } + + def get_host_info(): """ Returns host information such as the host name and network interfaces. @@ -925,16 +878,16 @@ def get_host_info(): by_ipv4 = host_info["network"]["interfaces"]["by-ipv4"] by_ipv6 = host_info["network"]["interfaces"]["by-ipv6"] - ifaces = netifaces.interfaces() + ifaces = netinfo.netdev_info() for dev_name in ifaces: - addr_fams = netifaces.ifaddresses(dev_name) - af_link = addr_fams.get(netifaces.AF_LINK) - af_inet4 = addr_fams.get(netifaces.AF_INET) - af_inet6 = addr_fams.get(netifaces.AF_INET6) - - mac = None - if af_link and "addr" in af_link[0]: - mac = af_link[0]["addr"] + af_inet4 = [] + af_inet6 = [] + for addr in ifaces[dev_name]["ipv4"]: + af_inet4.append(convert_to_netifaces_format(addr)) + for addr in ifaces[dev_name]["ipv6"]: + af_inet6.append(convert_to_netifaces_format(addr)) + + mac = ifaces[dev_name].get("hwaddr") # Do not bother recording localhost if mac == "00:00:00:00:00:00": diff --git a/pyproject.toml b/pyproject.toml index 99854f397f8..6f8ccdd1348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ module = [ "debconf", "httplib", "jsonpatch", - "netifaces", "paramiko.*", "pip.*", "pycloudlib.*", diff --git a/requirements.txt b/requirements.txt index edec46a7fdd..eabd7a22cd8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,12 +29,3 @@ jsonpatch # For validating cloud-config sections per schema definitions jsonschema - -# Used by DataSourceVMware to inspect the host's network configuration during -# the "setup()" function. -# -# This allows a host that uses DHCP to bring up the network during BootLocal -# and still participate in instance-data by gathering the network in detail at -# runtime and merge that information into the metadata and repersist that to -# disk. -netifaces>=0.10.4 diff --git a/tests/unittests/sources/test_vmware.py b/tests/unittests/sources/test_vmware.py index 585f4fbdaad..33193f89f5e 100644 --- a/tests/unittests/sources/test_vmware.py +++ b/tests/unittests/sources/test_vmware.py @@ -63,6 +63,45 @@ - echo "Hello, world." """ +VMW_IPV4_ROUTEINFO = { + "destination": "0.0.0.0", + "flags": "G", + "gateway": "10.85.130.1", + "genmask": "0.0.0.0", + "iface": "eth0", + "metric": "50", +} +VMW_IPV4_NETDEV_ADDR = { + "bcast": "10.85.130.255", + "ip": "10.85.130.116", + "mask": "255.255.255.0", + "scope": "global", +} +VMW_IPV6_ROUTEINFO = { + "destination": "::/0", + "flags": "UG", + "gateway": "2001:67c:1562:8007::1", + "iface": "eth0", + "metric": "50", +} +VMW_IPV6_NETDEV_ADDR = { + "ip": "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64", + "scope6": "global", +} + + +def generate_test_netdev_data(ipv4=None, ipv6=None): + ipv4 = ipv4 or [] + ipv6 = ipv6 or [] + return { + "eth0": { + "hwaddr": "00:16:3e:16:db:54", + "ipv4": ipv4, + "ipv6": ipv6, + "up": True, + }, + } + @pytest.fixture(autouse=True) def common_patches(): @@ -74,8 +113,8 @@ def common_patches(): is_FreeBSD=mock.Mock(return_value=False), ), mock.patch( - "cloudinit.sources.DataSourceVMware.netifaces.interfaces", - return_value=[], + "cloudinit.netinfo.netdev_info", + return_value={}, ), mock.patch( "cloudinit.sources.DataSourceVMware.getfqdn", @@ -152,6 +191,124 @@ def test_get_host_info_dual(self, m_fn_ipaddr): host_info[DataSourceVMware.LOCAL_IPV6] == "2001:db8::::::8888" ) + # TODO migrate this entire test suite to pytest then parameterize + @mock.patch("cloudinit.netinfo.route_info") + @mock.patch("cloudinit.netinfo.netdev_info") + def test_get_default_ip_addrs_ipv4only( + self, + m_netdev_info, + m_route_info, + ): + """Test get_default_ip_addrs use cases""" + m_route_info.return_value = { + "ipv4": [VMW_IPV4_ROUTEINFO], + "ipv6": [], + } + m_netdev_info.return_value = generate_test_netdev_data( + ipv4=[VMW_IPV4_NETDEV_ADDR] + ) + ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() + self.assertEqual(ipv4, "10.85.130.116") + self.assertEqual(ipv6, None) + + @mock.patch("cloudinit.netinfo.route_info") + @mock.patch("cloudinit.netinfo.netdev_info") + def test_get_default_ip_addrs_ipv6only( + self, + m_netdev_info, + m_route_info, + ): + m_route_info.return_value = { + "ipv4": [], + "ipv6": [VMW_IPV6_ROUTEINFO], + } + m_netdev_info.return_value = generate_test_netdev_data( + ipv6=[VMW_IPV6_NETDEV_ADDR] + ) + ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() + self.assertEqual(ipv4, None) + self.assertEqual(ipv6, "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64") + + @mock.patch("cloudinit.netinfo.route_info") + @mock.patch("cloudinit.netinfo.netdev_info") + def test_get_default_ip_addrs_dualstack( + self, + m_netdev_info, + m_route_info, + ): + m_route_info.return_value = { + "ipv4": [VMW_IPV4_ROUTEINFO], + "ipv6": [VMW_IPV6_ROUTEINFO], + } + m_netdev_info.return_value = generate_test_netdev_data( + ipv4=[VMW_IPV4_NETDEV_ADDR], + ipv6=[VMW_IPV6_NETDEV_ADDR], + ) + ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() + self.assertEqual(ipv4, "10.85.130.116") + self.assertEqual(ipv6, "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64") + + @mock.patch("cloudinit.netinfo.route_info") + @mock.patch("cloudinit.netinfo.netdev_info") + def test_get_default_ip_addrs_multiaddr( + self, + m_netdev_info, + m_route_info, + ): + m_route_info.return_value = { + "ipv4": [VMW_IPV4_ROUTEINFO], + "ipv6": [], + } + m_netdev_info.return_value = generate_test_netdev_data( + ipv4=[ + VMW_IPV4_NETDEV_ADDR, + { + "bcast": "10.85.131.255", + "ip": "10.85.131.117", + "mask": "255.255.255.0", + "scope": "global", + }, + ], + ipv6=[ + VMW_IPV6_NETDEV_ADDR, + { + "ip": "fe80::216:3eff:fe16:db54/64", + "scope6": "link", + }, + ], + ) + ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() + self.assertEqual(ipv4, None) + self.assertEqual(ipv6, None) + + @mock.patch("cloudinit.netinfo.route_info") + @mock.patch("cloudinit.netinfo.netdev_info") + def test_get_default_ip_addrs_nodefault( + self, + m_netdev_info, + m_route_info, + ): + m_route_info.return_value = { + "ipv4": [ + { + "destination": "185.125.188.0", + "flags": "G", + "gateway": "10.85.130.1", + "genmask": "0.0.0.255", + "iface": "eth0", + "metric": "50", + }, + ], + "ipv6": [], + } + m_netdev_info.return_value = generate_test_netdev_data( + ipv4=[VMW_IPV4_NETDEV_ADDR], + ipv6=[VMW_IPV6_NETDEV_ADDR], + ) + ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() + self.assertEqual(ipv4, None) + self.assertEqual(ipv6, None) + @mock.patch("cloudinit.sources.DataSourceVMware.get_host_info") def test_wait_on_network(self, m_fn): metadata = { diff --git a/tools/build-on-netbsd b/tools/build-on-netbsd index 0d4eb58be4d..b743d591b6e 100755 --- a/tools/build-on-netbsd +++ b/tools/build-on-netbsd @@ -19,7 +19,6 @@ pkgs=" ${py_prefix}-oauthlib ${py_prefix}-requests ${py_prefix}-setuptools - ${py_prefix}-netifaces ${py_prefix}-yaml ${py_prefix}-jsonschema sudo diff --git a/tools/build-on-openbsd b/tools/build-on-openbsd index bc551c0da44..93a9d501676 100755 --- a/tools/build-on-openbsd +++ b/tools/build-on-openbsd @@ -16,7 +16,6 @@ pkgs=" py3-configobj py3-jinja2 py3-jsonschema - py3-netifaces py3-oauthlib py3-requests py3-setuptools diff --git a/tox.ini b/tox.ini index ba3bc8a8fb3..cad37a1d746 100644 --- a/tox.ini +++ b/tox.ini @@ -194,7 +194,6 @@ deps = requests==2.18.4 jsonpatch==1.16 jsonschema==2.6.0 - netifaces==0.10.4 # test-requirements pytest==3.3.2 pytest-cov==2.5.1