diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index b70099519..83f0c1a15 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -25,3 +25,5 @@ class Device(): model: str = None test_modules: str = None ip_addr: str = None + device_folder: str = None + max_device_reports: int = None diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index a0f6118ff..7306d5cc8 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -25,6 +25,7 @@ MONITOR_PERIOD_KEY = 'monitor_period' STARTUP_TIMEOUT_KEY = 'startup_timeout' LOG_LEVEL_KEY = 'log_level' +MAX_DEVICE_REPORTS_KEY = 'max_device_reports' class TestRunSession(): """Represents the current session of Test Run.""" @@ -62,7 +63,8 @@ def _get_default_config(self): 'log_level': 'INFO', 'startup_timeout': 60, 'monitor_period': 30, - 'runtime': 120 + 'runtime': 120, + 'max_device_reports': 5 } def get_config(self): @@ -95,6 +97,9 @@ def _load_config(self): if LOG_LEVEL_KEY in config_file_json: self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY) + if MAX_DEVICE_REPORTS_KEY in config_file_json: + self._config[MAX_DEVICE_REPORTS_KEY] = config_file_json.get(MAX_DEVICE_REPORTS_KEY) + def _save_config(self): with open(self._config_file, 'w', encoding='utf-8') as f: f.write(json.dumps(self._config, indent=2)) @@ -116,6 +121,9 @@ def get_monitor_period(self): def get_startup_timeout(self): return self._config.get(STARTUP_TIMEOUT_KEY) + + def get_max_device_reports(self): + return self._config.get(MAX_DEVICE_REPORTS_KEY) def set_config(self, config_json): self._config = config_json diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 6016fbfe7..0c3de6db4 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -53,6 +53,7 @@ DEVICE_MODEL = 'model' DEVICE_MAC_ADDR = 'mac_addr' DEVICE_TEST_MODULES = 'test_modules' +MAX_DEVICE_REPORTS_KEY = 'max_device_reports' class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. @@ -112,7 +113,15 @@ def _load_devices(self, device_dir): util.run_command(f'chown -R {util.get_host_user()} {device_dir}') for device_folder in os.listdir(device_dir): - with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), + + device_config_file_path = os.path.join(device_dir, + device_folder, + DEVICE_CONFIG) + if not os.path.exists(device_config_file_path): + LOGGER.error(f'Device configuration file missing from device {device_folder}') + continue + + with open(device_config_file_path, encoding='utf-8') as device_config_file: device_config_json = json.load(device_config_file) @@ -120,11 +129,18 @@ def _load_devices(self, device_dir): device_model = device_config_json.get(DEVICE_MODEL) mac_addr = device_config_json.get(DEVICE_MAC_ADDR) test_modules = device_config_json.get(DEVICE_TEST_MODULES) + max_device_reports = None + if 'max_device_reports' in device_config_json: + max_device_reports = device_config_json.get(MAX_DEVICE_REPORTS_KEY) device = Device(manufacturer=device_manufacturer, model=device_model, mac_addr=mac_addr, - test_modules=test_modules) + test_modules=test_modules, + max_device_reports=max_device_reports, + device_folder=device_folder) + self.get_session().add_device(device) + self.get_session().add_device(device) LOGGER.debug(f'Loaded device {device.manufacturer} {device.model} with MAC address {device.mac_addr}') diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 7d550d4ae..4c56a05f0 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -154,17 +154,25 @@ def _device_discovered(self, mac_addr): # Ignore device if not registered return - device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, - mac_addr.replace(':', '')) - os.makedirs(device_runtime_dir) + device_runtime_dir = os.path.join(RUNTIME_DIR, + TEST_DIR, + mac_addr.replace(':', '') + ) + + # Cleanup any old current test files + shutil.rmtree(device_runtime_dir, ignore_errors=True) + os.makedirs(device_runtime_dir, exist_ok=True) + util.run_command(f'chown -R {self._host_user} {device_runtime_dir}') packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_startup_timeout(), stop_filter=self._device_has_ip) wrpcap( - os.path.join(RUNTIME_DIR, TEST_DIR, mac_addr.replace(':', ''), - 'startup.pcap'), packet_capture) + os.path.join(device_runtime_dir, + 'startup.pcap' + ), + packet_capture) if device.ip_addr is None: LOGGER.info( @@ -201,14 +209,23 @@ def _start_device_monitor(self, device): callback the steady state method for this device.""" LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' f'for {str(self._session.get_monitor_period())} seconds') + + device_runtime_dir = os.path.join(RUNTIME_DIR, + TEST_DIR, + device.mac_addr.replace(':', '') + ) packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_monitor_period()) wrpcap( - os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), - 'monitor.pcap'), packet_capture) + os.path.join(device_runtime_dir, + 'monitor.pcap' + ), + packet_capture) self._monitor_in_progress = False - self.get_listener().call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) + self.get_listener().call_callback( + NetworkEvent.DEVICE_STABLE, + device.mac_addr) def _check_network_services(self): LOGGER.debug('Checking network modules...') diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 61b94a995..cfa4b6e29 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Provides high level management of the test orchestrator.""" import os import json import time import shutil import docker +from datetime import datetime from docker.types import Mount from common import logger, util from test_orc.module import TestModule @@ -27,6 +27,7 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" +SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports" DEVICE_ROOT_CERTS = "local/root_certs" @@ -35,24 +36,18 @@ class TestOrchestrator: def __init__(self, session, net_orc): self._test_modules = [] - self._module_config = None self._session = session self._net_orc = net_orc self._test_in_progress = False + self._path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - self._path = os.path.dirname(os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - - # Resolve the path to the test-run folder - #self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) - - self._root_path = os.path.dirname(os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - - shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR), - ignore_errors=True) + self._root_path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) def start(self): LOGGER.debug("Starting test orchestrator") @@ -82,38 +77,128 @@ def run_test_modules(self, device): LOGGER.info("All tests complete") self._generate_results(device) + self._timestamp_results(device) + LOGGER.debug("Cleaning old test results...") + self._cleanup_old_test_results(device) + LOGGER.debug("Old test results cleaned") self._test_in_progress = False + def _cleanup_old_test_results(self, device): + + if device.max_device_reports is not None: + max_device_reports = device.max_device_reports + else: + max_device_reports = self._session.get_max_device_reports() + + completed_results_dir = os.path.join( + self._root_path, + SAVED_DEVICE_REPORTS.replace("{device_folder}", + device.device_folder) + ) + + completed_tests = os.listdir(completed_results_dir) + cur_test_count = len(completed_tests) + if cur_test_count > max_device_reports: + LOGGER.debug("Current device has more than max tests results allowed: " + + str(cur_test_count) + ">" + str(max_device_reports)) + + # Find and delete the oldest test + oldest_test = self._find_oldest_test(completed_results_dir) + if oldest_test is not None: + LOGGER.debug("Oldest test found, removing: " + str(oldest_test)) + shutil.rmtree(oldest_test, ignore_errors=True) + # Confirm the delete was succesful + new_test_count = len(os.listdir(completed_results_dir)) + if (new_test_count != cur_test_count + and new_test_count > max_device_reports): + # Continue cleaning up until we're under the max + self._cleanup_old_test_results(device) + + def _find_oldest_test(self, completed_tests_dir): + oldest_timestamp = None + oldest_directory = None + for completed_test in os.listdir(completed_tests_dir): + timestamp = datetime.strptime(str(completed_test), "%Y-%m-%dT%H:%M:%S") + if oldest_timestamp is None or timestamp < oldest_timestamp: + oldest_timestamp = timestamp + oldest_directory = completed_test + if oldest_directory: + return os.path.join(completed_tests_dir, oldest_directory) + else: + return None + + def _timestamp_results(self, device): + + # Define the current device results directory + cur_results_dir = os.path.join( + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", "") + ) + + # Define the destination results directory with timestamp + cur_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + completed_results_dir = os.path.join( + SAVED_DEVICE_REPORTS.replace("{device_folder}", + device.device_folder), + cur_time) + + # Copy the results to the timestamp directory + # leave current copy in place for quick reference to + # most recent test + shutil.copytree(cur_results_dir, completed_results_dir) + util.run_command(f"chown -R {self._host_user} '{completed_results_dir}'") + def _generate_results(self, device): - results = {} - results["device"] = {} + + report = {} + + report["device"] = {} if device.manufacturer is not None: - results["device"]["manufacturer"] = device.manufacturer + report["device"]["manufacturer"] = device.manufacturer if device.model is not None: - results["device"]["model"] = device.model - results["device"]["mac_addr"] = device.mac_addr + report["device"]["model"] = device.model + report["device"]["mac_addr"] = device.mac_addr + + results = [] + for module in self._test_modules: if module.enable_container and self._is_module_enabled(module, device): + container_runtime_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/" + module.name) + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", ""), + module.name + ) + results_file = f"{container_runtime_dir}/{module.name}-result.json" try: with open(results_file, "r", encoding="utf-8-sig") as f: module_results = json.load(f) - results[module.name] = module_results + for result in module_results["results"]: + results.append(result) except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: - LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") + LOGGER.error( + f("Error occured whilst obbtaining results " + "for module {module.name}") + ) LOGGER.debug(results_error) + report["results"] = results + out_file = os.path.join( - self._root_path, - "runtime/test/" + device.mac_addr.replace(":", "") + "/results.json") + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", ""), + "report.json" + ) + with open(out_file, "w", encoding="utf-8") as f: - json.dump(results, f, indent=2) + json.dump(report, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") - return results + return report def test_in_progress(self): return self._test_in_progress @@ -139,21 +224,31 @@ def _run_test_module(self, module, device): LOGGER.info("Running test module " + module.name) try: + + device_test_dir = os.path.join( + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", "") + ) + container_runtime_dir = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/" + module.name) - os.makedirs(container_runtime_dir) + device_test_dir, + module.name + ) + os.makedirs(container_runtime_dir, exist_ok=True) network_runtime_dir = os.path.join(self._root_path, "runtime/network") device_startup_capture = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/startup.pcap") + device_test_dir, + "startup.pcap" + ) util.run_command(f"chown -R {self._host_user} {device_startup_capture}") device_monitor_capture = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/monitor.pcap") + device_test_dir, + "monitor.pcap" + ) util.run_command(f"chown -R {self._host_user} {device_monitor_capture}") client = docker.from_env() @@ -206,13 +301,12 @@ def _run_test_module(self, module, device): status = self._get_module_status(module) log_stream = module.container.logs(stream=True, stdout=True, stderr=True) - while (time.time() < test_module_timeout and - status == "running" and - self._session.get_status() == "In progress"): + while (time.time() < test_module_timeout and status == "running" + and self._session.get_status() == "In progress"): try: line = next(log_stream).decode("utf-8").strip() print(line) - except Exception: + except Exception: # pylint: disable=W0718 time.sleep(1) status = self._get_module_status(module) diff --git a/local/.gitignore b/local/.gitignore index d3086d4df..06f79c1ca 100644 --- a/local/.gitignore +++ b/local/.gitignore @@ -1,3 +1,3 @@ -system.json -devices -root_certs +system.json +devices +root_certs diff --git a/local/system.json.example b/local/system.json.example index e99e013f3..17e5b0891 100644 --- a/local/system.json.example +++ b/local/system.json.example @@ -6,5 +6,6 @@ "log_level": "INFO", "startup_timeout": 60, "monitor_period": 300, - "runtime": 1200 + "runtime": 1200, + "max_device_reports": 5 } \ No newline at end of file diff --git a/resources/devices/template/device_config.json b/resources/devices/template/device_config.json index 7ee63cf95..ac8ff197c 100644 --- a/resources/devices/template/device_config.json +++ b/resources/devices/template/device_config.json @@ -2,6 +2,7 @@ "manufacturer": "Manufacturer X", "model": "Device X", "mac_addr": "aa:bb:cc:dd:ee:ff", + "max_device_tests":5, "test_modules": { "dns": { "enabled": true