From dd37cfa3f495f6f29687e68eb1cec70aa4bf0fe2 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 10 Aug 2023 16:34:28 -0600 Subject: [PATCH 1/4] Fix network request from module config Misc formatting issues in test orchestrator --- .../python/src/test_orc/test_orchestrator.py | 74 ++++++++----------- 1 file changed, 31 insertions(+), 43 deletions(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 7a7d19bdb..b9353c995 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -28,7 +28,7 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" -LOG_REGEX = r'^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_' +LOG_REGEX = r"^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_" SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports" DEVICE_ROOT_CERTS = "local/root_certs" @@ -79,7 +79,7 @@ def run_test_modules(self): for module in self._test_modules: self._run_test_module(module) LOGGER.info("All tests complete") - + self._session.stop() self._generate_report() self._test_in_progress = False @@ -96,16 +96,17 @@ def _generate_report(self): report = {} report["device"] = self._session.get_target_device().to_json() - report["started"] = self._session.get_started().strftime("%Y-%m-%d %H:%M:%S") - report["finished"] = self._session.get_finished().strftime("%Y-%m-%d %H:%M:%S") + report["started"] = self._session.get_started().strftime( + "%Y-%m-%d %H:%M:%S") + report["finished"] = self._session.get_finished().strftime( + "%Y-%m-%d %H:%M:%S") report["status"] = self._session.get_status() report["results"] = self._session.get_test_results() out_file = os.path.join( - self._root_path, - RUNTIME_DIR, + self._root_path, RUNTIME_DIR, self._session.get_target_device().mac_addr.replace(":", ""), "report.json") - + with open(out_file, "w", encoding="utf-8") as f: json.dump(report, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") @@ -119,10 +120,8 @@ def _cleanup_old_test_results(self, device): max_device_reports = self._session.get_max_device_reports() completed_results_dir = os.path.join( - self._root_path, - SAVED_DEVICE_REPORTS.replace("{device_folder}", - device.device_folder) - ) + self._root_path, + SAVED_DEVICE_REPORTS.replace("{device_folder}", device.device_folder)) completed_tests = os.listdir(completed_results_dir) cur_test_count = len(completed_tests) @@ -138,7 +137,7 @@ def _cleanup_old_test_results(self, device): # Confirm the delete was succesful new_test_count = len(os.listdir(completed_results_dir)) if (new_test_count != cur_test_count - and new_test_count > max_device_reports): + and new_test_count > max_device_reports): # Continue cleaning up until we're under the max self._cleanup_old_test_results(device) @@ -158,18 +157,14 @@ def _find_oldest_test(self, completed_tests_dir): def _timestamp_results(self, device): # Define the current device results directory - cur_results_dir = os.path.join( - self._root_path, - RUNTIME_DIR, - device.mac_addr.replace(":", "") - ) + cur_results_dir = os.path.join(self._root_path, RUNTIME_DIR, + device.mac_addr.replace(":", "")) # Define the destination results directory with timestamp cur_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") completed_results_dir = os.path.join( - SAVED_DEVICE_REPORTS.replace("{device_folder}", - device.device_folder), - cur_time) + SAVED_DEVICE_REPORTS.replace("{device_folder}", device.device_folder), + cur_time) # Copy the results to the timestamp directory # leave current copy in place for quick reference to @@ -204,30 +199,18 @@ def _run_test_module(self, module): try: - device_test_dir = os.path.join( - self._root_path, - RUNTIME_DIR, - device.mac_addr.replace(":", "") - ) - - container_runtime_dir = os.path.join( - device_test_dir, - module.name - ) + device_test_dir = os.path.join(self._root_path, RUNTIME_DIR, + device.mac_addr.replace(":", "")) + + container_runtime_dir = os.path.join(device_test_dir, module.name) os.makedirs(container_runtime_dir, exist_ok=True) network_runtime_dir = os.path.join(self._root_path, "runtime/network") - device_startup_capture = os.path.join( - device_test_dir, - "startup.pcap" - ) + device_startup_capture = os.path.join(device_test_dir, "startup.pcap") util.run_command(f"chown -R {self._host_user} {device_startup_capture}") - device_monitor_capture = os.path.join( - device_test_dir, - "monitor.pcap" - ) + device_monitor_capture = os.path.join(device_test_dir, "monitor.pcap") util.run_command(f"chown -R {self._host_user} {device_monitor_capture}") client = docker.from_env() @@ -286,24 +269,25 @@ def _run_test_module(self, module): line = next(log_stream).decode("utf-8").strip() if re.search(LOG_REGEX, line): print(line) - except Exception: # pylint: disable=W0718 + except Exception: # pylint: disable=W0718 time.sleep(1) status = self._get_module_status(module) # Get test results from module container_runtime_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/" + module.name) + self._root_path, + "runtime/test/" + device.mac_addr.replace(":", "") + "/" + module.name) results_file = f"{container_runtime_dir}/{module.name}-result.json" try: with open(results_file, "r", encoding="utf-8-sig") as f: module_results_json = json.load(f) - module_results = module_results_json['results'] + module_results = module_results_json["results"] for test_result in module_results: self._session.add_test_result(test_result) except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: - LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") + LOGGER.error( + f"Error occured whilst obbtaining results for module {module.name}") LOGGER.debug(results_error) LOGGER.info("Test module " + module.name + " has finished") @@ -379,6 +363,10 @@ def _load_test_module(self, module_dir): module.enable_container = module_json["config"]["docker"][ "enable_container"] + # Determine if this module needs network access + if "network" in module_json["config"]: + module.network = module_json["config"]["network"] + if "depends_on" in module_json["config"]["docker"]: depends_on_module = module_json["config"]["docker"]["depends_on"] if self._get_test_module(depends_on_module) is None: From 1fae76fdcd409adce54a97fed55fc6a419de54ba Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 10 Aug 2023 17:04:15 -0600 Subject: [PATCH 2/4] fix misc network orchestrator formatting issues --- .../src/net_orc/network_orchestrator.py | 92 +++++++++---------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index ebeeba2dd..19cf0081a 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Network orchestrator is responsible for managing all of the virtual network services""" import ipaddress @@ -44,13 +43,11 @@ PRIVATE_DOCKER_NET = 'tr-private-net' CONTAINER_NAME = 'network_orchestrator' + class NetworkOrchestrator: """Manage and controls a virtual testing network.""" - def __init__(self, - session, - validate=True, - single_intf=False): + def __init__(self, session, validate=True, single_intf=False): self._session = session self._monitor_in_progress = False @@ -103,8 +100,9 @@ def check_config(self): return False else: if not device_interface_ready and not internet_interface_ready: - LOGGER.error('Both device and internet interfaces are not ready for use. ' + - 'Ensure both interfaces are connected.') + LOGGER.error( + 'Both device and internet interfaces are not ready for use. ' + + 'Ensure both interfaces are connected.') return False elif not device_interface_ready: LOGGER.error('Device interface is not ready for use. ' + @@ -169,14 +167,13 @@ def _device_discovered(self, mac_addr): f'Discovered device {mac_addr}. Waiting for device to obtain IP') if device is None: - LOGGER.debug(f'Device with MAC address {mac_addr} does not exist in device repository') + LOGGER.debug(f'Device with MAC address {mac_addr} does not exist' + + ' in device repository') # Ignore device if not registered return - device_runtime_dir = os.path.join(RUNTIME_DIR, - TEST_DIR, - mac_addr.replace(':', '') - ) + device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, + mac_addr.replace(':', '')) # Cleanup any old current test files shutil.rmtree(device_runtime_dir, ignore_errors=True) @@ -187,11 +184,7 @@ def _device_discovered(self, mac_addr): packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_startup_timeout(), stop_filter=self._device_has_ip) - wrpcap( - os.path.join(device_runtime_dir, - 'startup.pcap' - ), - packet_capture) + wrpcap(os.path.join(device_runtime_dir, 'startup.pcap'), packet_capture) if device.ip_addr is None: LOGGER.info( @@ -228,23 +221,17 @@ def _start_device_monitor(self, device): callback the steady state method for this device.""" LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' f'for {str(self._session.get_monitor_period())} seconds') - - device_runtime_dir = os.path.join(RUNTIME_DIR, - TEST_DIR, - device.mac_addr.replace(':', '') - ) - - packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_monitor_period()) - wrpcap( - os.path.join(device_runtime_dir, - 'monitor.pcap' - ), - packet_capture) + + device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, + device.mac_addr.replace(':', '')) + + packet_capture = sniff(iface=self._session.get_device_interface(), + timeout=self._session.get_monitor_period()) + wrpcap(os.path.join(device_runtime_dir, 'monitor.pcap'), packet_capture) self._monitor_in_progress = False - self.get_listener().call_callback( - NetworkEvent.DEVICE_STABLE, - device.mac_addr) + self.get_listener().call_callback(NetworkEvent.DEVICE_STABLE, + device.mac_addr) def _check_network_services(self): LOGGER.debug('Checking network modules...') @@ -297,24 +284,32 @@ def _ci_pre_network_create(self): 'ip route | head -n 1 | awk \'{print $3}\'', shell=True).decode('utf-8').strip() self._ipv4 = subprocess.check_output( - f'ip a show {self._session.get_internet_interface()} | grep \"inet \" | awk \'{{print $2}}\'', + (f'ip a show {self._session.get_internet_interface()} | ' + + 'grep \"inet \" | awk \'{{print $2}}\''), shell=True).decode('utf-8').strip() self._ipv6 = subprocess.check_output( - f'ip a show {self._session.get_internet_interface()} | grep inet6 | awk \'{{print $2}}\'', + (f'ip a show {self._session.get_internet_interface()} | grep inet6 | ' + + 'awk \'{{print $2}}\''), shell=True).decode('utf-8').strip() self._brd = subprocess.check_output( - f'ip a show {self._session.get_internet_interface()} | grep \"inet \" | awk \'{{print $4}}\'', + (f'ip a show {self._session.get_internet_interface()} | grep \"inet \" ' + + '| awk \'{{print $4}}\''), shell=True).decode('utf-8').strip() def _ci_post_network_create(self): """ Restore network connection in CI environment """ LOGGER.info('post cr') - util.run_command(f'ip address del {self._ipv4} dev {self._session.get_internet_interface()}') - util.run_command(f'ip -6 address del {self._ipv6} dev {self._session.get_internet_interface()}') + util.run_command(((f'ip address del {self._ipv4} ' + + 'dev {self._session.get_internet_interface()}'))) + util.run_command((f'ip -6 address del {self._ipv6} ' + + 'dev {self._session.get_internet_interface()}')) util.run_command( - f'ip link set dev {self._session.get_internet_interface()} address 00:B0:D0:63:C2:26') - util.run_command(f'ip addr flush dev {self._session.get_internet_interface()}') - util.run_command(f'ip addr add dev {self._session.get_internet_interface()} 0.0.0.0') + (f'ip link set dev {self._session.get_internet_interface()} ' + + 'address 00:B0:D0:63:C2:26')) + util.run_command( + f'ip addr flush dev {self._session.get_internet_interface()}') + util.run_command( + f'ip addr add dev {self._session.get_internet_interface()} 0.0.0.0') util.run_command( f'ip addr add dev {INTERNET_BRIDGE} {self._ipv4} broadcast {self._brd}') util.run_command(f'ip -6 addr add {self._ipv6} dev {INTERNET_BRIDGE} ') @@ -331,7 +326,7 @@ def create_net(self): # TODO: This is not just for CI #if self._single_intf: - #self._ci_pre_network_create() + #self._ci_pre_network_create() # Setup the virtual network if not self._ovs.create_baseline_net(verify=True): @@ -341,15 +336,15 @@ def create_net(self): # TODO: This is not just for CI #if self._single_intf: - #self._ci_post_network_create() + #self._ci_post_network_create() self._create_private_net() self._listener = Listener(self._session) self.get_listener().register_callback(self._device_discovered, - [NetworkEvent.DEVICE_DISCOVERED]) + [NetworkEvent.DEVICE_DISCOVERED]) self.get_listener().register_callback(self._dhcp_lease_ack, - [NetworkEvent.DHCP_LEASE_ACK]) + [NetworkEvent.DHCP_LEASE_ACK]) def load_network_modules(self): """Load network modules from module_config.json.""" @@ -624,7 +619,7 @@ def _attach_service_to_network(self, net_module): # Add and configure the interface container if not self._ip_ctrl.configure_container_interface( - bridge_intf, container_intf, "veth0", container_net_ns, mac_addr, + bridge_intf, container_intf, 'veth0', container_net_ns, mac_addr, net_module.container_name, ipv4_addr, ipv6_addr): LOGGER.error('Failed to configure local networking for ' + net_module.name + '. Exiting.') @@ -650,7 +645,7 @@ def _attach_service_to_network(self, net_module): container_intf = 'tr-cti-' + net_module.dir_name if not self._ip_ctrl.configure_container_interface( - bridge_intf, container_intf, "eth1", container_net_ns, mac_addr): + bridge_intf, container_intf, 'eth1', container_net_ns, mac_addr): LOGGER.error('Failed to configure internet networking for ' + net_module.name + '. Exiting.') sys.exit(1) @@ -667,7 +662,8 @@ def restore_net(self): LOGGER.info('Clearing baseline network') - if hasattr(self, 'listener') and self.get_listener() is not None and self.get_listener().is_running(): + if hasattr(self, 'listener') and self.get_listener( + ) is not None and self.get_listener().is_running(): self.get_listener().stop_listener() client = docker.from_env() @@ -719,6 +715,7 @@ def __init__(self): self.net_config = NetworkModuleNetConfig() + class NetworkModuleNetConfig: """Define all the properties of the network config for a network module""" @@ -741,6 +738,7 @@ def get_ipv4_addr_with_prefix(self): def get_ipv6_addr_with_prefix(self): return format(self.ipv6_address) + '/' + str(self.ipv6_network.prefixlen) + class NetworkConfig: """Define all the properties of the network configuration""" From c8f4aea6772a8e09d4a4817def6af843603fc3b5 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 10 Aug 2023 17:09:10 -0600 Subject: [PATCH 3/4] fix misc ovs control formatting issues --- framework/python/src/net_orc/ovs_control.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/framework/python/src/net_orc/ovs_control.py b/framework/python/src/net_orc/ovs_control.py index a2769632c..80f76e85f 100644 --- a/framework/python/src/net_orc/ovs_control.py +++ b/framework/python/src/net_orc/ovs_control.py @@ -11,10 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """OVS Control Module""" -import json -import os from common import logger from common import util @@ -80,12 +77,14 @@ def validate_baseline_network(self): int_bridge = True # Verify the device bridge - dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._session.get_device_interface()]) + dev_bridge = self.verify_bridge(DEVICE_BRIDGE, + [self._session.get_device_interface()]) LOGGER.debug('Device bridge verified: ' + str(dev_bridge)) # Verify the internet bridge if 'single_intf' not in self._session.get_runtime_params(): - int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()]) + int_bridge = self.verify_bridge(INTERNET_BRIDGE, + [self._session.get_internet_interface()]) LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) return dev_bridge and int_bridge @@ -118,7 +117,8 @@ def create_baseline_net(self, verify=True): # Remove IP from internet adapter if not 'single_intf' in self._session.get_runtime_params(): - self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0') + self.set_interface_ip(interface=self._session.get_internet_interface(), + ip_addr='0.0.0.0') self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE) # Enable forwarding of eapol packets From 75965d3c7bb19c6659ee9f1ef909da86c7200631 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 10 Aug 2023 17:17:24 -0600 Subject: [PATCH 4/4] fix misc ip control formatting issues --- framework/python/src/net_orc/ip_control.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index eb683c46b..5c9f86d18 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -34,7 +34,7 @@ def add_link(self, interface_name, peer_name): def add_namespace(self, namespace): """Add a network namespace""" exists = self.namespace_exists(namespace) - LOGGER.info("Namespace exists: " + str(exists)) + LOGGER.info('Namespace exists: ' + str(exists)) if exists: return True else: @@ -58,14 +58,11 @@ def link_exists(self, link_name): def namespace_exists(self, namespace): """Check if a namespace already exists""" namespaces = self.get_namespaces() - if namespace in namespaces: - return True - else: - return False + return namespace in namespaces def get_links(self): - stdout, stderr = util.run_command('ip link list') - links = stdout.strip().split('\n') + result = util.run_command('ip link list') + links = result[0].strip().split('\n') netns_links = [] for link in links: match = re.search(r'\d+:\s+(\S+)', link) @@ -78,9 +75,9 @@ def get_links(self): return netns_links def get_namespaces(self): - stdout, stderr = util.run_command('ip netns list') + result = util.run_command('ip netns list') #Strip ID's from the namespace results - namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', stdout) + namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', result[0]) return namespaces def set_namespace(self, interface_name, namespace): @@ -187,9 +184,8 @@ def configure_container_interface(self, # Rename container interface name if not self.rename_interface(container_intf, namespace, namespace_intf): - LOGGER.error( - f'Failed to rename container interface {container_intf} to {namespace_intf}' - ) + LOGGER.error((f'Failed to rename container interface {container_intf} ' + + 'to {namespace_intf}')) return False # Set MAC address of container interface