From 2001d9b5eddc6ee933035107229e74bde2eb5cf5 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 3 Aug 2023 11:30:18 +0100 Subject: [PATCH 1/3] Keep results in memory --- framework/python/src/api/api.py | 10 ++- framework/python/src/common/device.py | 10 +++ framework/python/src/common/session.py | 23 ++++-- framework/python/src/core/testrun.py | 42 +++++++---- .../src/net_orc/network_orchestrator.py | 29 +++++--- framework/python/src/net_orc/ovs_control.py | 20 ++--- .../python/src/test_orc/test_orchestrator.py | 74 +++++++++++-------- modules/test/base/bin/capture | 2 +- modules/test/base/bin/setup_binaries | 2 +- modules/test/base/bin/start | 2 +- 10 files changed, 139 insertions(+), 75 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index d877a5b33..f63f1825a 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -108,11 +108,14 @@ async def start_test_run(self, request: Request, response: Response): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid JSON received") - if "device" not in body_json or "mac_addr" not in body_json["device"]: + if "device" not in body_json or not ( + "mac_addr" in body_json["device"] and + "firmware" in body_json["device"]): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid request received") device = self._session.get_device(body_json["device"]["mac_addr"]) + device.firmware = body_json["device"]["firmware"] # Check Test Run is not already running if self._test_run.get_session().get_status() != "Idle": @@ -123,12 +126,13 @@ async def start_test_run(self, request: Request, response: Response): # Check if requested device is known in the device repository if device is None: response.status_code = status.HTTP_404_NOT_FOUND - return self._generate_msg(False, "A device with that MAC address could not be found") + return self._generate_msg(False, + "A device with that MAC address could not be found") # Check Test Run is able to start if self._test_run.get_net_orc().check_config() is False: response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR - return self._generate_msg(False, "Configured interfaces are not ready for use. Ensure both interfaces are connected.") + return self._generate_msg(False,"Configured interfaces are not ready for use. Ensure required interfaces are connected.") self._test_run.get_session().set_target_device(device) LOGGER.info(f"Starting Test Run with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}") diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index b70099519..745436c23 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -25,3 +25,13 @@ class Device(): model: str = None test_modules: str = None ip_addr: str = None + firmware: str = None + + def to_json(self): + device_json = {} + device_json['mac_addr'] = self.mac_addr + device_json['manufacturer'] = self.manufacturer + device_json['model'] = self.model + if self.firmware is not None: + device_json['firmware'] = self.firmware + return device_json diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index a0f6118ff..751ca6144 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -34,7 +34,8 @@ def __init__(self, config_file): self._device = None self._started = None self._finished = None - self._tests = [] + self._results = [] + self._runtime_params = [] self._config_file = config_file @@ -53,6 +54,9 @@ def get_started(self): def get_finished(self): return self._finished + def stop(self): + self._finished = datetime.datetime.now() + def _get_default_config(self): return { 'network': { @@ -105,6 +109,12 @@ def get_runtime(self): def get_log_level(self): return self._config.get(LOG_LEVEL_KEY) + def get_runtime_params(self): + return self._runtime_params + + def add_runtime_param(self, param): + self._runtime_params.append(param) + def get_device_interface(self): return self._config.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) @@ -149,13 +159,16 @@ def get_status(self): def set_status(self, status): self._status = status - def get_tests(self): - return self._tests + def get_test_results(self): + return self._results + + def add_test_result(self, test_result): + self._results.append(test_result) def reset(self): self.set_status('Idle') self.set_target_device(None) - self._tests = [] + self._results = [] self._started = None self._finished = None @@ -165,5 +178,5 @@ def to_json(self): 'device': self.get_target_device(), 'started': self.get_started(), 'finished': self.get_finished(), - 'tests': self.get_tests() + 'results': self.get_test_results() } diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 6016fbfe7..6ce0b9ca0 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -80,7 +80,14 @@ def __init__(self, # Catch any exit signals self._register_exits() + # Create session self._session = TestRunSession(config_file=self._config_file) + + if single_intf: + self._session.add_runtime_param('single_intf') + if net_only: + self._session.add_runtime_param('net_only') + self._load_all_devices() self._net_orc = net_orc.NetworkOrchestrator( @@ -92,6 +99,12 @@ def __init__(self, self._net_orc) if self._no_ui: + # Check Test Run is able to start + if self.get_net_orc().check_config() is False: + return + + # Any additional checks that need to be performed go here + self.start() else: self._api = Api(self) @@ -176,7 +189,7 @@ def start(self): self.get_net_orc().monitor_in_progress()): time.sleep(5) - self.stop() + self.stop() def stop(self, kill=False): self._set_status('Stopping') @@ -238,27 +251,26 @@ def get_device(self, mac_addr): def _device_discovered(self, mac_addr): - if self.get_session().get_target_device() is not None: - if mac_addr != self.get_session().get_target_device().mac_addr: - # Ignore discovered device - return + device = self.get_session().get_target_device() - self._set_status('Identifying device') - device = self.get_device(mac_addr) if device is not None: - LOGGER.info( - f'Discovered {device.manufacturer} {device.model} on the network') + if mac_addr != device.mac_addr: + # Ignore discovered device because it is not the target device + return else: - device = Device(mac_addr=mac_addr) - self._devices.append(device) - LOGGER.info( - f'A new device has been discovered with mac address {mac_addr}') + device = self.get_device(mac_addr) + if device is None: + return + + self.get_session().set_target_device(device) + + LOGGER.info( + f'Discovered {device.manufacturer} {device.model} on the network') def _device_stable(self, mac_addr): - device = self.get_device(mac_addr) LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') self._set_status('In progress') - self._test_orc.run_test_modules(device) + self._test_orc.run_test_modules() self._set_status('Complete') def _set_status(self, status): diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 7d550d4ae..befa15d65 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -90,10 +90,20 @@ def start(self): def check_config(self): - if not util.interface_exists(self._session.get_internet_interface()) or not util.interface_exists( - self._session.get_device_interface()): + interfaces_ready = True + if 'single_intf' in self._session.get_runtime_params(): + # Check for device interface only + interfaces_ready = util.interface_exists( + self._session.get_device_interface()) + else: + # Check for both + interfaces_ready = util.interface_exists( + self._session.get_device_interface()) and util.interface_exists( + self._session.get_internet_interface()) + + if not interfaces_ready: LOGGER.error('Configured interfaces are not ready for use. ' + - 'Ensure both interfaces are connected.') + 'Ensure required interfaces are connected.') return False return True @@ -293,11 +303,9 @@ def _ci_post_network_create(self): def create_net(self): LOGGER.info('Creating baseline network') - if self._single_intf: - self._ci_pre_network_create() - - # Remove IP from internet adapter - util.run_command('ifconfig ' + self._session.get_internet_interface() + ' 0.0.0.0') + # TODO: This is not just for CI + #if self._single_intf: + #self._ci_pre_network_create() # Setup the virtual network if not self._ovs.create_baseline_net(verify=True): @@ -305,8 +313,9 @@ def create_net(self): self.stop() sys.exit(1) - if self._single_intf: - self._ci_post_network_create() + # TODO: This is not just for CI + #if self._single_intf: + #self._ci_post_network_create() self._create_private_net() diff --git a/framework/python/src/net_orc/ovs_control.py b/framework/python/src/net_orc/ovs_control.py index c48e58e3b..a2769632c 100644 --- a/framework/python/src/net_orc/ovs_control.py +++ b/framework/python/src/net_orc/ovs_control.py @@ -76,13 +76,17 @@ def validate_baseline_network(self): # Verify the OVS setup of the virtual network LOGGER.debug('Validating baseline network') + dev_bridge = True + int_bridge = True + # Verify the device bridge dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._session.get_device_interface()]) LOGGER.debug('Device bridge verified: ' + str(dev_bridge)) # Verify the internet bridge - int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()]) - LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) + if 'single_intf' not in self._session.get_runtime_params(): + int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()]) + LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) return dev_bridge and int_bridge @@ -103,21 +107,19 @@ def verify_bridge(self, bridge_name, ports): def create_baseline_net(self, verify=True): LOGGER.debug('Creating baseline network') - # Remove IP from internet adapter - self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0') - # Create data plane self.add_bridge(DEVICE_BRIDGE) # Create control plane self.add_bridge(INTERNET_BRIDGE) - # Remove IP from internet adapter - self.set_interface_ip(self._session.get_internet_interface(), '0.0.0.0') - # Add external interfaces to data and control plane self.add_port(self._session.get_device_interface(), DEVICE_BRIDGE) - self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE) + + # Remove IP from internet adapter + if not 'single_intf' in self._session.get_runtime_params(): + self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0') + self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE) # Enable forwarding of eapol packets self.add_flow(bridge_name=DEVICE_BRIDGE, diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 74e399df1..be9610549 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -15,6 +15,7 @@ """Provides high level management of the test orchestrator.""" import os import json +import re import time import shutil import docker @@ -27,6 +28,7 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" +LOG_REGEX = r'^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_' class TestOrchestrator: @@ -68,48 +70,41 @@ def stop(self): """Stop any running tests""" self._stop_modules() - def run_test_modules(self, device): + def run_test_modules(self): """Iterates through each test module and starts the container.""" + + device = self._session.get_target_device() self._test_in_progress = True LOGGER.info( f"Running test modules on device with mac addr {device.mac_addr}") for module in self._test_modules: - self._run_test_module(module, device) + self._run_test_module(module) LOGGER.info("All tests complete") - self._generate_results(device) + self._session.stop() + self._generate_report() self._test_in_progress = False - def _generate_results(self, device): - results = {} - results["device"] = {} - if device.manufacturer is not None: - results["device"]["manufacturer"] = device.manufacturer - if device.model is not None: - results["device"]["model"] = device.model - results["device"]["mac_addr"] = device.mac_addr - for module in self._test_modules: - if module.enable_container and self._is_module_enabled(module, device): - container_runtime_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/" + module.name) - results_file = f"{container_runtime_dir}/{module.name}-result.json" - try: - with open(results_file, "r", encoding="utf-8-sig") as f: - module_results = json.load(f) - results[module.name] = module_results - except (FileNotFoundError, PermissionError, - json.JSONDecodeError) as results_error: - LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") - LOGGER.debug(results_error) + def _generate_report(self): + + # TODO: Calculate the status result + # We need to know the required result of each test + report = {} + report["device"] = self._session.get_target_device().to_json() + report["started"] = self._session.get_started().strftime("%Y-%m-%d %H:%M:%S") + report["finished"] = self._session.get_finished().strftime("%Y-%m-%d %H:%M:%S") + report["status"] = self._session.get_status() + report["results"] = self._session.get_test_results() out_file = os.path.join( self._root_path, - "runtime/test/" + device.mac_addr.replace(":", "") + "/results.json") + "runtime/test/" + + self._session.get_target_device().mac_addr.replace(":", "") + + "/report.json") with open(out_file, "w", encoding="utf-8") as f: - json.dump(results, f, indent=2) + json.dump(report, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") - return results + return report def test_in_progress(self): return self._test_in_progress @@ -123,9 +118,11 @@ def _is_module_enabled(self, module, device): enabled = test_modules[module.name]["enabled"] return enabled - def _run_test_module(self, module, device): + def _run_test_module(self, module): """Start the test container and extract the results.""" + device = self._session.get_target_device() + if module is None or not module.enable_container: return @@ -207,11 +204,28 @@ def _run_test_module(self, module, device): self._session.get_status() == "In progress"): try: line = next(log_stream).decode("utf-8").strip() - print(line) + if re.search(LOG_REGEX, line): + print(line) except Exception: time.sleep(1) status = self._get_module_status(module) + # Get test results from module + container_runtime_dir = os.path.join( + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + "/" + module.name) + results_file = f"{container_runtime_dir}/{module.name}-result.json" + try: + with open(results_file, "r", encoding="utf-8-sig") as f: + module_results_json = json.load(f) + module_results = module_results_json['results'] + for test_result in module_results: + self._session.add_test_result(test_result) + except (FileNotFoundError, PermissionError, + json.JSONDecodeError) as results_error: + LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") + LOGGER.debug(results_error) + LOGGER.info("Test module " + module.name + " has finished") def _get_module_status(self, module): diff --git a/modules/test/base/bin/capture b/modules/test/base/bin/capture index e237f3d72..69fa916c3 100644 --- a/modules/test/base/bin/capture +++ b/modules/test/base/bin/capture @@ -27,7 +27,7 @@ INTERFACE=$2 # Create the output directory and start the capture mkdir -p $PCAP_DIR chown $HOST_USER $PCAP_DIR -tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & 2>&1 /dev/null +tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & # Small pause to let the capture to start sleep 1 \ No newline at end of file diff --git a/modules/test/base/bin/setup_binaries b/modules/test/base/bin/setup_binaries index eaccf9de6..6af744693 100644 --- a/modules/test/base/bin/setup_binaries +++ b/modules/test/base/bin/setup_binaries @@ -18,7 +18,7 @@ BIN_DIR=$1 # Remove incorrect line endings -dos2unix $BIN_DIR/* >/dev/null 2>&1 +dos2unix $BIN_DIR/* # Make sure all the bin files are executable chmod u+x $BIN_DIR/* \ No newline at end of file diff --git a/modules/test/base/bin/start b/modules/test/base/bin/start index 6869d1116..37902b868 100755 --- a/modules/test/base/bin/start +++ b/modules/test/base/bin/start @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -/testrun/bin/start_module > /dev/null \ No newline at end of file +/testrun/bin/start_module \ No newline at end of file From 959a3c03fd8b29d3e3f567f3fef3d98be57cbb8f Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 3 Aug 2023 17:35:19 +0100 Subject: [PATCH 2/3] More useful debug message --- .../src/net_orc/network_orchestrator.py | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index befa15d65..e38728cb1 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -90,21 +90,30 @@ def start(self): def check_config(self): - interfaces_ready = True - if 'single_intf' in self._session.get_runtime_params(): - # Check for device interface only - interfaces_ready = util.interface_exists( + device_interface_ready = util.interface_exists( self._session.get_device_interface()) - else: - # Check for both - interfaces_ready = util.interface_exists( - self._session.get_device_interface()) and util.interface_exists( + internet_interface_ready = util.interface_exists( self._session.get_internet_interface()) - if not interfaces_ready: - LOGGER.error('Configured interfaces are not ready for use. ' + - 'Ensure required interfaces are connected.') - return False + if 'single_intf' in self._session.get_runtime_params(): + # Check for device interface only + if not device_interface_ready: + LOGGER.error('Device interface is not ready for use. ' + + 'Ensure device interface is connected.') + return False + else: + if not device_interface_ready and not internet_interface_ready: + LOGGER.error('Both device and internet interfaces are not ready for use. ' + + 'Ensure both interfaces are connected.') + return False + elif not device_interface_ready: + LOGGER.error('Device interface is not ready for use. ' + + 'Ensure device interface is connected.') + return False + elif not internet_interface_ready: + LOGGER.error('Internet interface is not ready for use. ' + + 'Ensure internet interface is connected.') + return False return True def start_network(self): From dc302370596f659a5cbbb315a0b2187514a80593 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 3 Aug 2023 17:55:47 +0100 Subject: [PATCH 3/3] Fix file path --- framework/python/src/test_orc/test_orchestrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 6fb1266c1..7a7d19bdb 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -102,7 +102,7 @@ def _generate_report(self): report["results"] = self._session.get_test_results() out_file = os.path.join( self._root_path, - RUNTIME_DIR + + RUNTIME_DIR, self._session.get_target_device().mac_addr.replace(":", ""), "report.json")