From 30cad447dacb0b96a2102c912ea080b7dc007c84 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 26 Aug 2020 08:12:30 +0000 Subject: [PATCH 1/8] Make azure parse_network_config non-fatal on exception --- cloudinit/sources/DataSourceAzure.py | 143 ++++++++++-------- tests/unittests/test_datasource/test_azure.py | 33 +++- 2 files changed, 107 insertions(+), 69 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f3c6452b931..1cdd55dc8f1 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1338,76 +1338,97 @@ def load_azure_ds_dir(source_dir): return (md, ud, cfg, {'ovf-env.xml': contents}) -def parse_network_config(imds_metadata): +@azure_ds_telemetry_reporter +def parse_network_config(imds_metadata) -> dict: """Convert imds_metadata dictionary to network v2 configuration. - Parses network configuration from imds metadata if present or generate fallback network config excluding mlx4_core devices. @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - with events.ReportEventStack( - name="parse_network_config", - description="", - parent=azure_ds_reporter - ) as evt: + netconfig = {} + try: if imds_metadata != sources.UNSET and imds_metadata: - netconfig = {'version': 2, 'ethernets': {}} - LOG.debug('Azure: generating network configuration from IMDS') - network_metadata = imds_metadata['network'] - for idx, intf in enumerate(network_metadata['interface']): - # First IPv4 and/or IPv6 address will be obtained via DHCP. - # Any additional IPs of each type will be set as static - # addresses. - nicname = 'eth{idx}'.format(idx=idx) - dhcp_override = {'route-metric': (idx + 1) * 100} - dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override, - 'dhcp6': False} - for addr_type in ('ipv4', 'ipv6'): - addresses = intf.get(addr_type, {}).get('ipAddress', []) - if addr_type == 'ipv4': - default_prefix = '24' - else: - default_prefix = '128' - if addresses: - dev_config['dhcp6'] = True - # non-primary interfaces should have a higher - # route-metric (cost) so default routes prefer - # primary nic due to lower route-metric value - dev_config['dhcp6-overrides'] = dhcp_override - for addr in addresses[1:]: - # Append static address config for ip > 1 - netPrefix = intf[addr_type]['subnet'][0].get( - 'prefix', default_prefix) - privateIp = addr['privateIpAddress'] - if not dev_config.get('addresses'): - dev_config['addresses'] = [] - dev_config['addresses'].append( - '{ip}/{prefix}'.format( - ip=privateIp, prefix=netPrefix)) - if dev_config: - mac = ':'.join(re.findall(r'..', intf['macAddress'])) - dev_config.update({ - 'match': {'macaddress': mac.lower()}, - 'set-name': nicname - }) - # With netvsc, we can get two interfaces that - # share the same MAC, so we need to make sure - # our match condition also contains the driver - driver = device_driver(nicname) - if driver and driver == 'hv_netvsc': - dev_config['match']['driver'] = driver - netconfig['ethernets'][nicname] = dev_config - evt.description = "network config from imds" + netconfig = _generate_network_config_from_imds_metadata( + imds_metadata) else: - blacklist = ['mlx4_core'] - LOG.debug('Azure: generating fallback configuration') - # generate a network config, blacklist picking mlx4_core devs - netconfig = net.generate_fallback_config( - blacklist_drivers=blacklist, config_driver=True) - evt.description = "network config from fallback" - return netconfig + netconfig = _generate_network_config_from_fallback_config() + except Exception as e: + msg = 'Failed generating network config: %s' % e + report_diagnostic_event(msg) + LOG.warning(msg) + return netconfig + + +@azure_ds_telemetry_reporter +def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: + """Convert imds_metadata dictionary to network v2 configuration. + Parses network configuration from imds metadata. + + @param: imds_metadata: Dict of content read from IMDS network service. + @return: Dictionary containing network version 2 standard configuration. + """ + netconfig = {'version': 2, 'ethernets': {}} + LOG.debug('Azure: generating network configuration from IMDS') + network_metadata = imds_metadata['network'] + for idx, intf in enumerate(network_metadata['interface']): + # First IPv4 and/or IPv6 address will be obtained via DHCP. + # Any additional IPs of each type will be set as static + # addresses. + nicname = 'eth{idx}'.format(idx=idx) + dhcp_override = {'route-metric': (idx + 1) * 100} + dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override, + 'dhcp6': False} + for addr_type in ('ipv4', 'ipv6'): + addresses = intf.get(addr_type, {}).get('ipAddress', []) + if addr_type == 'ipv4': + default_prefix = '24' + else: + default_prefix = '128' + if addresses: + dev_config['dhcp6'] = True + # non-primary interfaces should have a higher + # route-metric (cost) so default routes prefer + # primary nic due to lower route-metric value + dev_config['dhcp6-overrides'] = dhcp_override + for addr in addresses[1:]: + # Append static address config for ip > 1 + netPrefix = intf[addr_type]['subnet'][0].get( + 'prefix', default_prefix) + privateIp = addr['privateIpAddress'] + if not dev_config.get('addresses'): + dev_config['addresses'] = [] + dev_config['addresses'].append( + '{ip}/{prefix}'.format( + ip=privateIp, prefix=netPrefix)) + if dev_config: + mac = ':'.join(re.findall(r'..', intf['macAddress'])) + dev_config.update({ + 'match': {'macaddress': mac.lower()}, + 'set-name': nicname + }) + # With netvsc, we can get two interfaces that + # share the same MAC, so we need to make sure + # our match condition also contains the driver + driver = device_driver(nicname) + if driver and driver == 'hv_netvsc': + dev_config['match']['driver'] = driver + netconfig['ethernets'][nicname] = dev_config + return netconfig + + +@azure_ds_telemetry_reporter +def _generate_network_config_from_fallback_config() -> dict: + """Generate fallback network config excluding mlx4_core devices. + + @return: Dictionary containing network version 2 standard configuration. + """ + blacklist = ['mlx4_core'] + report_diagnostic_event('Azure: generating fallback configuration') + # generate a network config, blacklist picking mlx4_core devs + return net.generate_fallback_config( + blacklist_drivers=blacklist, config_driver=True) @azure_ds_telemetry_reporter diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 47e03bd1a74..c12b4f288ee 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -157,7 +157,9 @@ class TestParseNetworkConfig(CiTestCase): maxDiff = None - def test_single_ipv4_nic_configuration(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_single_ipv4_nic_configuration(self, m_driver): """parse_network_config emits dhcp on single nic with ipv4""" expected = {'ethernets': { 'eth0': {'dhcp4': True, @@ -167,7 +169,9 @@ def test_single_ipv4_nic_configuration(self): 'set-name': 'eth0'}}, 'version': 2} self.assertEqual(expected, dsaz.parse_network_config(NETWORK_METADATA)) - def test_increases_route_metric_for_non_primary_nics(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_increases_route_metric_for_non_primary_nics(self, m_driver): """parse_network_config increases route-metric for each nic""" expected = {'ethernets': { 'eth0': {'dhcp4': True, @@ -194,7 +198,9 @@ def test_increases_route_metric_for_non_primary_nics(self): imds_data['network']['interface'].append(third_intf) self.assertEqual(expected, dsaz.parse_network_config(imds_data)) - def test_ipv4_and_ipv6_route_metrics_match_for_nics(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_ipv4_and_ipv6_route_metrics_match_for_nics(self, m_driver): """parse_network_config emits matching ipv4 and ipv6 route-metrics.""" expected = {'ethernets': { 'eth0': {'addresses': ['10.0.0.5/24', '2001:dead:beef::2/128'], @@ -236,7 +242,9 @@ def test_ipv4_and_ipv6_route_metrics_match_for_nics(self): imds_data['network']['interface'].append(third_intf) self.assertEqual(expected, dsaz.parse_network_config(imds_data)) - def test_ipv4_secondary_ips_will_be_static_addrs(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_ipv4_secondary_ips_will_be_static_addrs(self, m_driver): """parse_network_config emits primary ipv4 as dhcp others are static""" expected = {'ethernets': { 'eth0': {'addresses': ['10.0.0.5/24'], @@ -256,7 +264,9 @@ def test_ipv4_secondary_ips_will_be_static_addrs(self): } self.assertEqual(expected, dsaz.parse_network_config(imds_data)) - def test_ipv6_secondary_ips_will_be_static_cidrs(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_ipv6_secondary_ips_will_be_static_cidrs(self, m_driver): """parse_network_config emits primary ipv6 as dhcp others are static""" expected = {'ethernets': { 'eth0': {'addresses': ['10.0.0.5/24', '2001:dead:beef::2/10'], @@ -777,7 +787,9 @@ def test_user_cfg_set_agent_command_plain(self): self.assertTrue(ret) self.assertEqual(data['agent_invoked'], cfg['agent_command']) - def test_network_config_set_from_imds(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_network_config_set_from_imds(self, m_driver): """Datasource.network_config returns IMDS network data.""" sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} odata = {} @@ -795,7 +807,10 @@ def test_network_config_set_from_imds(self): dsrc.get_data() self.assertEqual(expected_network_config, dsrc.network_config) - def test_network_config_set_from_imds_route_metric_for_secondary_nic(self): + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_network_config_set_from_imds_route_metric_for_secondary_nic( + self, m_driver): """Datasource.network_config adds route-metric to secondary nics.""" sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} odata = {} @@ -1151,8 +1166,10 @@ def test_list_possible_azure_ds_devs(self, m_check_fbsd_cdrom, self.assertEqual( [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list) + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) @mock.patch('cloudinit.net.generate_fallback_config') - def test_imds_network_config(self, mock_fallback): + def test_imds_network_config(self, mock_fallback, m_driver): """Network config is generated from IMDS network data when present.""" sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} odata = {'HostName': "myhost", 'UserName': "myuser"} From f4f274f585b608fc93c4719b6465fd263c6986ec Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 26 Aug 2020 09:00:59 +0000 Subject: [PATCH 2/8] Make parse_network_config fall back on fallback network config on failure to generate network config from IMDS --- cloudinit/sources/DataSourceAzure.py | 22 +++++---- tests/unittests/test_datasource/test_azure.py | 45 +++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 1cdd55dc8f1..f9592f9172d 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1347,18 +1347,21 @@ def parse_network_config(imds_metadata) -> dict: @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - netconfig = {} - try: - if imds_metadata != sources.UNSET and imds_metadata: - netconfig = _generate_network_config_from_imds_metadata( + if imds_metadata != sources.UNSET and imds_metadata: + try: + return _generate_network_config_from_imds_metadata( imds_metadata) - else: - netconfig = _generate_network_config_from_fallback_config() + except Exception as e: + msg = 'Failed generating network config from IMDS metadata: %s' % e + report_diagnostic_event(msg) + LOG.warning(msg) + try: + return _generate_network_config_from_fallback_config() except Exception as e: - msg = 'Failed generating network config: %s' % e + msg = 'Failed generating fallback network config: %s' % e report_diagnostic_event(msg) LOG.warning(msg) - return netconfig + return {} @azure_ds_telemetry_reporter @@ -1370,7 +1373,8 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: @return: Dictionary containing network version 2 standard configuration. """ netconfig = {'version': 2, 'ethernets': {}} - LOG.debug('Azure: generating network configuration from IMDS') + report_diagnostic_event( + 'Azure: generating network configuration from IMDS') network_metadata = imds_metadata['network'] for idx, intf in enumerate(network_metadata['interface']): # First IPv4 and/or IPv6 address will be obtained via DHCP. diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index c12b4f288ee..9fe7d7d8c14 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -156,6 +156,15 @@ def construct_valid_ovf_env(data=None, pubkeys=None, class TestParseNetworkConfig(CiTestCase): maxDiff = None + fallback_config = { + 'version': 1, + 'config': [{ + 'type': 'physical', 'name': 'eth0', + 'mac_address': '00:11:22:33:44:55', + 'params': {'driver': 'hv_netsvc'}, + 'subnets': [{'type': 'dhcp'}], + }] + } @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', return_value=None) @@ -305,6 +314,42 @@ def test_match_driver_for_netvsc(self, m_driver): }}, 'version': 2} self.assertEqual(expected, dsaz.parse_network_config(NETWORK_METADATA)) + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + @mock.patch('cloudinit.net.generate_fallback_config') + def test_parse_network_config_uses_fallback_cfg_when_no_network_metadata( + self, m_fallback_config, m_driver): + """parse_network_config generates fallback network config when the + IMDS instance metadata is corrupted/invalid, such as when + network metadata is not present. + """ + imds_metadata_missing_network_metadata = copy.deepcopy( + NETWORK_METADATA) + del imds_metadata_missing_network_metadata['network'] + m_fallback_config.return_value = self.fallback_config + self.assertEqual( + self.fallback_config, + dsaz.parse_network_config( + imds_metadata_missing_network_metadata)) + + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + @mock.patch('cloudinit.net.generate_fallback_config') + def test_parse_network_config_uses_fallback_cfg_when_no_interface_metadata( + self, m_fallback_config, m_driver): + """parse_network_config generates fallback network config when the + IMDS instance metadata is corrupted/invalid, such as when + network interface metadata is not present. + """ + imds_metadata_missing_interface_metadata = copy.deepcopy( + NETWORK_METADATA) + del imds_metadata_missing_interface_metadata['network']['interface'] + m_fallback_config.return_value = self.fallback_config + self.assertEqual( + self.fallback_config, + dsaz.parse_network_config( + imds_metadata_missing_interface_metadata)) + class TestGetMetadataFromIMDS(HttprettyTestCase): From 8b280fd9e07d9545d1195022cec4d502e424949a Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 26 Aug 2020 18:53:35 +0000 Subject: [PATCH 3/8] Simplify reporting of diagnostic event --- cloudinit/sources/DataSourceAzure.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f9592f9172d..925b8514a90 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1347,21 +1347,16 @@ def parse_network_config(imds_metadata) -> dict: @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - if imds_metadata != sources.UNSET and imds_metadata: - try: - return _generate_network_config_from_imds_metadata( - imds_metadata) - except Exception as e: - msg = 'Failed generating network config from IMDS metadata: %s' % e - report_diagnostic_event(msg) - LOG.warning(msg) try: - return _generate_network_config_from_fallback_config() + if imds_metadata != sources.UNSET and imds_metadata: + return _generate_network_config_from_imds_metadata(imds_metadata) + else: + return _generate_network_config_from_fallback_config() except Exception as e: - msg = 'Failed generating fallback network config: %s' % e + msg = 'Failed generating network config: %s' % e report_diagnostic_event(msg) LOG.warning(msg) - return {} + return {} @azure_ds_telemetry_reporter @@ -1373,8 +1368,6 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: @return: Dictionary containing network version 2 standard configuration. """ netconfig = {'version': 2, 'ethernets': {}} - report_diagnostic_event( - 'Azure: generating network configuration from IMDS') network_metadata = imds_metadata['network'] for idx, intf in enumerate(network_metadata['interface']): # First IPv4 and/or IPv6 address will be obtained via DHCP. @@ -1429,7 +1422,6 @@ def _generate_network_config_from_fallback_config() -> dict: @return: Dictionary containing network version 2 standard configuration. """ blacklist = ['mlx4_core'] - report_diagnostic_event('Azure: generating fallback configuration') # generate a network config, blacklist picking mlx4_core devs return net.generate_fallback_config( blacklist_drivers=blacklist, config_driver=True) From 321d4eb842cc1b08b2cd93fd209a1c16434ee046 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 26 Aug 2020 19:19:34 +0000 Subject: [PATCH 4/8] Ensure that fallback config is generated when imds network config raises exc --- cloudinit/sources/DataSourceAzure.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 925b8514a90..3ab7a9a26a6 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1347,16 +1347,16 @@ def parse_network_config(imds_metadata) -> dict: @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - try: - if imds_metadata != sources.UNSET and imds_metadata: + if imds_metadata != sources.UNSET and imds_metadata: + try: return _generate_network_config_from_imds_metadata(imds_metadata) - else: - return _generate_network_config_from_fallback_config() - except Exception as e: - msg = 'Failed generating network config: %s' % e - report_diagnostic_event(msg) - LOG.warning(msg) - return {} + except Exception: + pass + try: + return _generate_network_config_from_fallback_config() + except Exception: + pass + return {} @azure_ds_telemetry_reporter From 54b38c74814737dec48fa99c8a55321377b6e0d6 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 26 Aug 2020 21:15:02 +0000 Subject: [PATCH 5/8] Log error on network config generation failure --- cloudinit/sources/DataSourceAzure.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 3ab7a9a26a6..c6638d20841 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1351,11 +1351,12 @@ def parse_network_config(imds_metadata) -> dict: try: return _generate_network_config_from_imds_metadata(imds_metadata) except Exception: - pass + LOG.error( + 'Failed generating network config from IMDS network metadata') try: return _generate_network_config_from_fallback_config() except Exception: - pass + LOG.error('Failed generating fallback network config') return {} From d5df930a008acd1358685905add43ea46d4e838e Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Sat, 29 Aug 2020 00:00:30 +0000 Subject: [PATCH 6/8] Blacklist mlx5_core devices from fallback network config --- cloudinit/sources/DataSourceAzure.py | 13 ++++++----- tests/unittests/test_datasource/test_azure.py | 22 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c6638d20841..049666c25cf 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1350,13 +1350,14 @@ def parse_network_config(imds_metadata) -> dict: if imds_metadata != sources.UNSET and imds_metadata: try: return _generate_network_config_from_imds_metadata(imds_metadata) - except Exception: + except Exception as e: LOG.error( - 'Failed generating network config from IMDS network metadata') + 'Failed generating network config ' + 'from IMDS network metadata: %s', str(e)) try: return _generate_network_config_from_fallback_config() - except Exception: - LOG.error('Failed generating fallback network config') + except Exception as e: + LOG.error('Failed generating fallback network config: %s', str(e)) return {} @@ -1418,11 +1419,11 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: @azure_ds_telemetry_reporter def _generate_network_config_from_fallback_config() -> dict: - """Generate fallback network config excluding mlx4_core devices. + """Generate fallback network config excluding mlx4_core & mlx5_core devices. @return: Dictionary containing network version 2 standard configuration. """ - blacklist = ['mlx4_core'] + blacklist = ['mlx4_core', 'mlx5_core'] # generate a network config, blacklist picking mlx4_core devs return net.generate_fallback_config( blacklist_drivers=blacklist, config_driver=True) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 9fe7d7d8c14..2f4d52fe9db 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1301,8 +1301,9 @@ def test_fallback_network_config(self, mock_fallback, mock_dd, netconfig = dsrc.network_config self.assertEqual(netconfig, fallback_config) - mock_fallback.assert_called_with(blacklist_drivers=['mlx4_core'], - config_driver=True) + mock_fallback.assert_called_with( + blacklist_drivers=['mlx4_core', 'mlx5_core'], + config_driver=True) @mock.patch('cloudinit.net.get_interface_mac') @mock.patch('cloudinit.net.get_devicelist') @@ -1324,19 +1325,15 @@ def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd, 'subnets': [{'type': 'dhcp'}], }] } - blacklist_config = { - 'type': 'physical', - 'name': 'eth1', - 'mac_address': '00:11:22:33:44:55', - 'params': {'driver': 'mlx4_core'} - } mock_fallback.return_value = fallback_config - mock_devlist.return_value = ['eth0', 'eth1'] + mock_devlist.return_value = ['eth0', 'eth1', 'eth2'] mock_dd.side_effect = [ 'hv_netsvc', # list composition, skipped 'mlx4_core', # list composition, match 'mlx4_core', # config get driver name + 'mlx5_core', # list composition, match + 'mlx5_core', # config get driver name ] mock_get_mac.return_value = '00:11:22:33:44:55' @@ -1347,9 +1344,10 @@ def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd, self.assertTrue(ret) netconfig = dsrc.network_config - expected_config = fallback_config - expected_config['config'].append(blacklist_config) - self.assertEqual(netconfig, expected_config) + self.assertEqual(netconfig, fallback_config) + mock_fallback.assert_called_with( + blacklist_drivers=['mlx4_core', 'mlx5_core'], + config_driver=True) @mock.patch(MOCKPATH + 'subp.subp') def test_get_hostname_with_no_args(self, m_subp): From 07bd46308699f7381c1ec63ff2114b29b25f0643 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 23 Sep 2020 20:30:43 +0000 Subject: [PATCH 7/8] Add comment on Azure VM Instance SKUs with mlx4 and mlx5 hardware --- cloudinit/sources/DataSourceAzure.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 654c1e377c6..773c60d7d16 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1472,8 +1472,17 @@ def _generate_network_config_from_fallback_config() -> dict: @return: Dictionary containing network version 2 standard configuration. """ + # Azure Dv4 and Ev4 series VMs always have mlx5 hardware. + # https://docs.microsoft.com/en-us/azure/virtual-machines/dv4-dsv4-series + # https://docs.microsoft.com/en-us/azure/virtual-machines/ev4-esv4-series + # Earlier D and E series VMs (such as Dv2, Dv3, and Ev3 series VMs) + # can have either mlx4 or mlx5 hardware, with the older series VMs + # having a higher chance of coming with mlx4 hardware. + # https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series + # https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series + # https://docs.microsoft.com/en-us/azure/virtual-machines/ev3-esv3-series blacklist = ['mlx4_core', 'mlx5_core'] - # generate a network config, blacklist picking mlx4_core devs + # generate a network config, blacklist picking mlx4_core and mlx5_core devs return net.generate_fallback_config( blacklist_drivers=blacklist, config_driver=True) From b64cd67a7c32be95a66855f8bd1ebaa00ecf9074 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 23 Sep 2020 20:34:46 +0000 Subject: [PATCH 8/8] Drop redundant test_azure UT test_fallback_network_config_blacklist --- tests/unittests/test_datasource/test_azure.py | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 050652af53a..2b22a87918f 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1311,50 +1311,6 @@ def test_fallback_network_config(self, mock_fallback, mock_dd, blacklist_drivers=['mlx4_core', 'mlx5_core'], config_driver=True) - @mock.patch('cloudinit.net.get_interface_mac') - @mock.patch('cloudinit.net.get_devicelist') - @mock.patch('cloudinit.net.device_driver') - @mock.patch('cloudinit.net.generate_fallback_config') - def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd, - mock_devlist, mock_get_mac): - """On absent network metadata, blacklist mlx from fallback config.""" - odata = {'HostName': "myhost", 'UserName': "myuser"} - data = {'ovfcontent': construct_valid_ovf_env(data=odata), - 'sys_cfg': {}} - - fallback_config = { - 'version': 1, - 'config': [{ - 'type': 'physical', 'name': 'eth0', - 'mac_address': '00:11:22:33:44:55', - 'params': {'driver': 'hv_netsvc'}, - 'subnets': [{'type': 'dhcp'}], - }] - } - mock_fallback.return_value = fallback_config - - mock_devlist.return_value = ['eth0', 'eth1', 'eth2'] - mock_dd.side_effect = [ - 'hv_netsvc', # list composition, skipped - 'mlx4_core', # list composition, match - 'mlx4_core', # config get driver name - 'mlx5_core', # list composition, match - 'mlx5_core', # config get driver name - ] - mock_get_mac.return_value = '00:11:22:33:44:55' - - dsrc = self._get_ds(data) - # Represent empty response from network imds - self.m_get_metadata_from_imds.return_value = {} - ret = dsrc.get_data() - self.assertTrue(ret) - - netconfig = dsrc.network_config - self.assertEqual(netconfig, fallback_config) - mock_fallback.assert_called_with( - blacklist_drivers=['mlx4_core', 'mlx5_core'], - config_driver=True) - @mock.patch(MOCKPATH + 'subp.subp') def test_get_hostname_with_no_args(self, m_subp): dsaz.get_hostname()