diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index d877a5b33..f63f1825a 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -108,11 +108,14 @@ async def start_test_run(self, request: Request, response: Response): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid JSON received") - if "device" not in body_json or "mac_addr" not in body_json["device"]: + if "device" not in body_json or not ( + "mac_addr" in body_json["device"] and + "firmware" in body_json["device"]): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid request received") device = self._session.get_device(body_json["device"]["mac_addr"]) + device.firmware = body_json["device"]["firmware"] # Check Test Run is not already running if self._test_run.get_session().get_status() != "Idle": @@ -123,12 +126,13 @@ async def start_test_run(self, request: Request, response: Response): # Check if requested device is known in the device repository if device is None: response.status_code = status.HTTP_404_NOT_FOUND - return self._generate_msg(False, "A device with that MAC address could not be found") + return self._generate_msg(False, + "A device with that MAC address could not be found") # Check Test Run is able to start if self._test_run.get_net_orc().check_config() is False: response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR - return self._generate_msg(False, "Configured interfaces are not ready for use. Ensure both interfaces are connected.") + return self._generate_msg(False,"Configured interfaces are not ready for use. Ensure required interfaces are connected.") self._test_run.get_session().set_target_device(device) LOGGER.info(f"Starting Test Run with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}") diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index 83f0c1a15..e2552d75a 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -25,5 +25,15 @@ class Device(): model: str = None test_modules: str = None ip_addr: str = None + firmware: str = None device_folder: str = None max_device_reports: int = None + + def to_json(self): + device_json = {} + device_json['mac_addr'] = self.mac_addr + device_json['manufacturer'] = self.manufacturer + device_json['model'] = self.model + if self.firmware is not None: + device_json['firmware'] = self.firmware + return device_json diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 7306d5cc8..13e4b09fb 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -35,7 +35,8 @@ def __init__(self, config_file): self._device = None self._started = None self._finished = None - self._tests = [] + self._results = [] + self._runtime_params = [] self._config_file = config_file @@ -54,6 +55,9 @@ def get_started(self): def get_finished(self): return self._finished + def stop(self): + self._finished = datetime.datetime.now() + def _get_default_config(self): return { 'network': { @@ -110,6 +114,12 @@ def get_runtime(self): def get_log_level(self): return self._config.get(LOG_LEVEL_KEY) + def get_runtime_params(self): + return self._runtime_params + + def add_runtime_param(self, param): + self._runtime_params.append(param) + def get_device_interface(self): return self._config.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) @@ -157,13 +167,16 @@ def get_status(self): def set_status(self, status): self._status = status - def get_tests(self): - return self._tests + def get_test_results(self): + return self._results + + def add_test_result(self, test_result): + self._results.append(test_result) def reset(self): self.set_status('Idle') self.set_target_device(None) - self._tests = [] + self._results = [] self._started = None self._finished = None @@ -173,5 +186,5 @@ def to_json(self): 'device': self.get_target_device(), 'started': self.get_started(), 'finished': self.get_finished(), - 'tests': self.get_tests() + 'results': self.get_test_results() } diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 0c3de6db4..6e3a6da5d 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -81,7 +81,14 @@ def __init__(self, # Catch any exit signals self._register_exits() + # Create session self._session = TestRunSession(config_file=self._config_file) + + if single_intf: + self._session.add_runtime_param('single_intf') + if net_only: + self._session.add_runtime_param('net_only') + self._load_all_devices() self._net_orc = net_orc.NetworkOrchestrator( @@ -93,6 +100,12 @@ def __init__(self, self._net_orc) if self._no_ui: + # Check Test Run is able to start + if self.get_net_orc().check_config() is False: + return + + # Any additional checks that need to be performed go here + self.start() else: self._api = Api(self) @@ -192,7 +205,7 @@ def start(self): self.get_net_orc().monitor_in_progress()): time.sleep(5) - self.stop() + self.stop() def stop(self, kill=False): self._set_status('Stopping') @@ -254,27 +267,26 @@ def get_device(self, mac_addr): def _device_discovered(self, mac_addr): - if self.get_session().get_target_device() is not None: - if mac_addr != self.get_session().get_target_device().mac_addr: - # Ignore discovered device - return + device = self.get_session().get_target_device() - self._set_status('Identifying device') - device = self.get_device(mac_addr) if device is not None: - LOGGER.info( - f'Discovered {device.manufacturer} {device.model} on the network') + if mac_addr != device.mac_addr: + # Ignore discovered device because it is not the target device + return else: - device = Device(mac_addr=mac_addr) - self._devices.append(device) - LOGGER.info( - f'A new device has been discovered with mac address {mac_addr}') + device = self.get_device(mac_addr) + if device is None: + return + + self.get_session().set_target_device(device) + + LOGGER.info( + f'Discovered {device.manufacturer} {device.model} on the network') def _device_stable(self, mac_addr): - device = self.get_device(mac_addr) LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') self._set_status('In progress') - self._test_orc.run_test_modules(device) + self._test_orc.run_test_modules() self._set_status('Complete') def _set_status(self, status): diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 4c56a05f0..ebeeba2dd 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -90,11 +90,30 @@ def start(self): def check_config(self): - if not util.interface_exists(self._session.get_internet_interface()) or not util.interface_exists( - self._session.get_device_interface()): - LOGGER.error('Configured interfaces are not ready for use. ' + - 'Ensure both interfaces are connected.') - return False + device_interface_ready = util.interface_exists( + self._session.get_device_interface()) + internet_interface_ready = util.interface_exists( + self._session.get_internet_interface()) + + if 'single_intf' in self._session.get_runtime_params(): + # Check for device interface only + if not device_interface_ready: + LOGGER.error('Device interface is not ready for use. ' + + 'Ensure device interface is connected.') + return False + else: + if not device_interface_ready and not internet_interface_ready: + LOGGER.error('Both device and internet interfaces are not ready for use. ' + + 'Ensure both interfaces are connected.') + return False + elif not device_interface_ready: + LOGGER.error('Device interface is not ready for use. ' + + 'Ensure device interface is connected.') + return False + elif not internet_interface_ready: + LOGGER.error('Internet interface is not ready for use. ' + + 'Ensure internet interface is connected.') + return False return True def start_network(self): @@ -310,11 +329,9 @@ def _ci_post_network_create(self): def create_net(self): LOGGER.info('Creating baseline network') - if self._single_intf: - self._ci_pre_network_create() - - # Remove IP from internet adapter - util.run_command('ifconfig ' + self._session.get_internet_interface() + ' 0.0.0.0') + # TODO: This is not just for CI + #if self._single_intf: + #self._ci_pre_network_create() # Setup the virtual network if not self._ovs.create_baseline_net(verify=True): @@ -322,8 +339,9 @@ def create_net(self): self.stop() sys.exit(1) - if self._single_intf: - self._ci_post_network_create() + # TODO: This is not just for CI + #if self._single_intf: + #self._ci_post_network_create() self._create_private_net() diff --git a/framework/python/src/net_orc/ovs_control.py b/framework/python/src/net_orc/ovs_control.py index c48e58e3b..a2769632c 100644 --- a/framework/python/src/net_orc/ovs_control.py +++ b/framework/python/src/net_orc/ovs_control.py @@ -76,13 +76,17 @@ def validate_baseline_network(self): # Verify the OVS setup of the virtual network LOGGER.debug('Validating baseline network') + dev_bridge = True + int_bridge = True + # Verify the device bridge dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._session.get_device_interface()]) LOGGER.debug('Device bridge verified: ' + str(dev_bridge)) # Verify the internet bridge - int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()]) - LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) + if 'single_intf' not in self._session.get_runtime_params(): + int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()]) + LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) return dev_bridge and int_bridge @@ -103,21 +107,19 @@ def verify_bridge(self, bridge_name, ports): def create_baseline_net(self, verify=True): LOGGER.debug('Creating baseline network') - # Remove IP from internet adapter - self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0') - # Create data plane self.add_bridge(DEVICE_BRIDGE) # Create control plane self.add_bridge(INTERNET_BRIDGE) - # Remove IP from internet adapter - self.set_interface_ip(self._session.get_internet_interface(), '0.0.0.0') - # Add external interfaces to data and control plane self.add_port(self._session.get_device_interface(), DEVICE_BRIDGE) - self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE) + + # Remove IP from internet adapter + if not 'single_intf' in self._session.get_runtime_params(): + self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0') + self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE) # Enable forwarding of eapol packets self.add_flow(bridge_name=DEVICE_BRIDGE, diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index cfa4b6e29..7a7d19bdb 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -14,6 +14,7 @@ """Provides high level management of the test orchestrator.""" import os import json +import re import time import shutil import docker @@ -27,6 +28,7 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" +LOG_REGEX = r'^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_' SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports" DEVICE_ROOT_CERTS = "local/root_certs" @@ -67,22 +69,48 @@ def stop(self): """Stop any running tests""" self._stop_modules() - def run_test_modules(self, device): + def run_test_modules(self): """Iterates through each test module and starts the container.""" + + device = self._session.get_target_device() self._test_in_progress = True LOGGER.info( f"Running test modules on device with mac addr {device.mac_addr}") for module in self._test_modules: - self._run_test_module(module, device) + self._run_test_module(module) LOGGER.info("All tests complete") - - self._generate_results(device) + + self._session.stop() + self._generate_report() + self._test_in_progress = False self._timestamp_results(device) LOGGER.debug("Cleaning old test results...") self._cleanup_old_test_results(device) LOGGER.debug("Old test results cleaned") self._test_in_progress = False + def _generate_report(self): + + # TODO: Calculate the status result + # We need to know the required result of each test + + report = {} + report["device"] = self._session.get_target_device().to_json() + report["started"] = self._session.get_started().strftime("%Y-%m-%d %H:%M:%S") + report["finished"] = self._session.get_finished().strftime("%Y-%m-%d %H:%M:%S") + report["status"] = self._session.get_status() + report["results"] = self._session.get_test_results() + out_file = os.path.join( + self._root_path, + RUNTIME_DIR, + self._session.get_target_device().mac_addr.replace(":", ""), + "report.json") + + with open(out_file, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + util.run_command(f"chown -R {self._host_user} {out_file}") + return report + def _cleanup_old_test_results(self, device): if device.max_device_reports is not None: @@ -149,57 +177,6 @@ def _timestamp_results(self, device): shutil.copytree(cur_results_dir, completed_results_dir) util.run_command(f"chown -R {self._host_user} '{completed_results_dir}'") - def _generate_results(self, device): - - report = {} - - report["device"] = {} - if device.manufacturer is not None: - report["device"]["manufacturer"] = device.manufacturer - if device.model is not None: - report["device"]["model"] = device.model - report["device"]["mac_addr"] = device.mac_addr - - results = [] - - for module in self._test_modules: - if module.enable_container and self._is_module_enabled(module, device): - - container_runtime_dir = os.path.join( - self._root_path, - RUNTIME_DIR, - device.mac_addr.replace(":", ""), - module.name - ) - - results_file = f"{container_runtime_dir}/{module.name}-result.json" - try: - with open(results_file, "r", encoding="utf-8-sig") as f: - module_results = json.load(f) - for result in module_results["results"]: - results.append(result) - except (FileNotFoundError, PermissionError, - json.JSONDecodeError) as results_error: - LOGGER.error( - f("Error occured whilst obbtaining results " - "for module {module.name}") - ) - LOGGER.debug(results_error) - - report["results"] = results - - out_file = os.path.join( - self._root_path, - RUNTIME_DIR, - device.mac_addr.replace(":", ""), - "report.json" - ) - - with open(out_file, "w", encoding="utf-8") as f: - json.dump(report, f, indent=2) - util.run_command(f"chown -R {self._host_user} {out_file}") - return report - def test_in_progress(self): return self._test_in_progress @@ -212,9 +189,11 @@ def _is_module_enabled(self, module, device): enabled = test_modules[module.name]["enabled"] return enabled - def _run_test_module(self, module, device): + def _run_test_module(self, module): """Start the test container and extract the results.""" + device = self._session.get_target_device() + if module is None or not module.enable_container: return @@ -305,11 +284,28 @@ def _run_test_module(self, module, device): and self._session.get_status() == "In progress"): try: line = next(log_stream).decode("utf-8").strip() - print(line) + if re.search(LOG_REGEX, line): + print(line) except Exception: # pylint: disable=W0718 time.sleep(1) status = self._get_module_status(module) + # Get test results from module + container_runtime_dir = os.path.join( + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + "/" + module.name) + results_file = f"{container_runtime_dir}/{module.name}-result.json" + try: + with open(results_file, "r", encoding="utf-8-sig") as f: + module_results_json = json.load(f) + module_results = module_results_json['results'] + for test_result in module_results: + self._session.add_test_result(test_result) + except (FileNotFoundError, PermissionError, + json.JSONDecodeError) as results_error: + LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") + LOGGER.debug(results_error) + LOGGER.info("Test module " + module.name + " has finished") def _get_module_status(self, module): diff --git a/modules/test/base/bin/capture b/modules/test/base/bin/capture index e237f3d72..69fa916c3 100644 --- a/modules/test/base/bin/capture +++ b/modules/test/base/bin/capture @@ -27,7 +27,7 @@ INTERFACE=$2 # Create the output directory and start the capture mkdir -p $PCAP_DIR chown $HOST_USER $PCAP_DIR -tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & 2>&1 /dev/null +tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & # Small pause to let the capture to start sleep 1 \ No newline at end of file diff --git a/modules/test/base/bin/setup_binaries b/modules/test/base/bin/setup_binaries index eaccf9de6..6af744693 100644 --- a/modules/test/base/bin/setup_binaries +++ b/modules/test/base/bin/setup_binaries @@ -18,7 +18,7 @@ BIN_DIR=$1 # Remove incorrect line endings -dos2unix $BIN_DIR/* >/dev/null 2>&1 +dos2unix $BIN_DIR/* # Make sure all the bin files are executable chmod u+x $BIN_DIR/* \ No newline at end of file diff --git a/modules/test/base/bin/start b/modules/test/base/bin/start index 6869d1116..37902b868 100755 --- a/modules/test/base/bin/start +++ b/modules/test/base/bin/start @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -/testrun/bin/start_module > /dev/null \ No newline at end of file +/testrun/bin/start_module \ No newline at end of file