From 42fc9c3196d070e16c3bd1ecdd75992410b72c68 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Sun, 16 Jul 2023 19:22:33 -0600 Subject: [PATCH 1/9] Change runtime test structure to allow for multiple old tests --- cmd/start | 3 - framework/python/src/core/device.py | 1 + .../src/net_orc/network_orchestrator.py | 12 ++- .../python/src/test_orc/test_orchestrator.py | 77 +++++++++++++++---- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/cmd/start b/cmd/start index 64ac197eb..d45fe1cc9 100755 --- a/cmd/start +++ b/cmd/start @@ -22,9 +22,6 @@ fi # Ensure that /var/run/netns folder exists mkdir -p /var/run/netns -# Clear up existing runtime files -rm -rf runtime - # Check if python modules exist. Install if not [ ! -d "venv" ] && cmd/install diff --git a/framework/python/src/core/device.py b/framework/python/src/core/device.py index efce2dba1..824517cd0 100644 --- a/framework/python/src/core/device.py +++ b/framework/python/src/core/device.py @@ -25,3 +25,4 @@ class Device(NetworkDevice): manufacturer: str = None model: str = None test_modules: str = None + max_device_tests: int = None diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 499ce954b..889f4caf3 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -164,8 +164,12 @@ def _device_discovered(self, mac_addr): device = self._get_device(mac_addr=mac_addr) device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, - device.mac_addr.replace(':', '')) - os.makedirs(device_runtime_dir) + device.mac_addr.replace(':', '') + '/current_test') + + # Cleanup any old current test files + shutil.rmtree(device_runtime_dir, ignore_errors=True) + os.makedirs(device_runtime_dir, exist_ok=True) + util.run_command(f'chown -R {self._host_user} {device_runtime_dir}') packet_capture = sniff(iface=self._dev_intf, @@ -173,7 +177,7 @@ def _device_discovered(self, mac_addr): stop_filter=self._device_has_ip) wrpcap( os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), - 'startup.pcap'), packet_capture) + 'current_test/startup.pcap'), packet_capture) if device.ip_addr is None: LOGGER.info( @@ -208,7 +212,7 @@ def _start_device_monitor(self, device): packet_capture = sniff(iface=self._dev_intf, timeout=self._monitor_period) wrpcap( os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), - 'monitor.pcap'), packet_capture) + 'current_test/monitor.pcap'), packet_capture) self._monitor_in_progress = False self.listener.call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index fef4e5bb5..6d9762cbd 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -18,6 +18,7 @@ import time import shutil import docker +from datetime import datetime from docker.types import Mount from common import logger, util from test_orc.module import TestModule @@ -37,22 +38,16 @@ def __init__(self, net_orc): self._module_config = None self._net_orc = net_orc self._test_in_progress = False + self._max_device_tests = 5 self._path = os.path.dirname(os.path.dirname( os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - # Resolve the path to the test-run folder - #self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) - - self._root_path = os.path.dirname(os.path.dirname( os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR), - ignore_errors=True) - def start(self): LOGGER.debug("Starting test orchestrator") @@ -78,8 +73,64 @@ def run_test_modules(self, device): LOGGER.info("All tests complete") self._generate_results(device) + self._timestamp_results(device) + LOGGER.info("Cleaning up old test results...") + self._cleanup_old_test_results(device) + LOGGER.info("Old tests cleaned up") self._test_in_progress = False + def _cleanup_old_test_results(self, device): + if device.max_device_tests is not None: + max_tests = device.max_device_tests + else: + max_tests = self._max_device_tests + completed_results_dir = cur_results_dir = os.path.join( + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + "/completed_tests/") + completed_tests = os.listdir(completed_results_dir) + cur_test_count = len(completed_tests) + if cur_test_count > max_tests: + LOGGER.debug("Current device has more than max tests results allowed: " + str(cur_test_count) + ">" + str(max_tests)) + # Find and delete the oldest test + oldest_test = self._find_oldest_test(completed_results_dir) + if oldest_test is not None: + LOGGER.debug("Oldest test found, removing: " + str(oldest_test)) + shutil.rmtree(oldest_test,ignore_errors=True) + # Confirm the delete was succesful + new_test_count = len(os.listdir(completed_results_dir)) + if new_test_count != cur_test_count and new_test_count > max_tests: + # Continue cleaning up until we're under the max + self._cleanup_old_test_results(device) + + def _find_oldest_test(self,completed_tests_dir): + oldest_timestamp = None + oldest_directory = None + for completed_test in os.listdir(completed_tests_dir): + timestamp = datetime.strptime(str(completed_test), '%Y-%m-%dT%H:%M:%S') + if oldest_timestamp is None or timestamp < oldest_timestamp: + oldest_timestamp = timestamp + oldest_directory = completed_test + if oldest_directory: + return os.path.join(completed_tests_dir, oldest_directory) + else: + return None + + + def _timestamp_results(self, device): + # Define the current device results directory + cur_results_dir = os.path.join( + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + "/current_test") + # Define the destination results directory with timestamp + cur_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + completed_results_dir = cur_results_dir = os.path.join( + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + "/completed_tests/" + cur_time) + # Move the results to the timestamp directory + os.makedirs(completed_results_dir, exist_ok=True) + shutil.move(cur_results_dir,completed_results_dir) + + def _generate_results(self, device): results = {} results["device"] = {} @@ -92,7 +143,7 @@ def _generate_results(self, device): if module.enable_container and self._is_module_enabled(module, device): container_runtime_dir = os.path.join( self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/" + module.name) + device.mac_addr.replace(":", "") + "/current_test/" + module.name) results_file = f"{container_runtime_dir}/{module.name}-result.json" try: with open(results_file, "r", encoding="utf-8-sig") as f: @@ -105,7 +156,7 @@ def _generate_results(self, device): out_file = os.path.join( self._root_path, - "runtime/test/" + device.mac_addr.replace(":", "") + "/results.json") + "runtime/test/" + device.mac_addr.replace(":", "") + "/current_test/results.json") with open(out_file, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") @@ -137,19 +188,19 @@ def _run_test_module(self, module, device): try: container_runtime_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/" + module.name) - os.makedirs(container_runtime_dir) + "/current_test/" + module.name) + os.makedirs(container_runtime_dir, exist_ok=True) network_runtime_dir = os.path.join(self._root_path, "runtime/network") device_startup_capture = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/startup.pcap") + "/current_test/startup.pcap") util.run_command(f"chown -R {self._host_user} {device_startup_capture}") device_monitor_capture = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/monitor.pcap") + "/current_test/monitor.pcap") util.run_command(f"chown -R {self._host_user} {device_monitor_capture}") client = docker.from_env() From 859d7b7944f5d14193a4edae47b6900ea55191c2 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Sun, 16 Jul 2023 20:05:01 -0600 Subject: [PATCH 2/9] fix current test move --- framework/python/src/test_orc/test_orchestrator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 6d9762cbd..36abe2e30 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -123,11 +123,10 @@ def _timestamp_results(self, device): device.mac_addr.replace(":", "") + "/current_test") # Define the destination results directory with timestamp cur_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') - completed_results_dir = cur_results_dir = os.path.join( + completed_results_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/completed_tests/" + cur_time) # Move the results to the timestamp directory - os.makedirs(completed_results_dir, exist_ok=True) shutil.move(cur_results_dir,completed_results_dir) From 2da6a8485c6a2f89aba1ab99d49d009595810c2d Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Sun, 16 Jul 2023 20:06:23 -0600 Subject: [PATCH 3/9] logging changes --- framework/python/src/test_orc/test_orchestrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 36abe2e30..3f60dda0b 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -74,9 +74,9 @@ def run_test_modules(self, device): self._generate_results(device) self._timestamp_results(device) - LOGGER.info("Cleaning up old test results...") + LOGGER.info("Cleaning old test results...") self._cleanup_old_test_results(device) - LOGGER.info("Old tests cleaned up") + LOGGER.info("Old test results cleaned") self._test_in_progress = False def _cleanup_old_test_results(self, device): From 8fe39b15c3c073d717babcae67c22c32e686698c Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Sun, 16 Jul 2023 20:18:34 -0600 Subject: [PATCH 4/9] Add device test count to device config --- framework/python/src/core/testrun.py | 7 ++++++- resources/devices/template/device_config.json | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index a91736e95..25b20f2d3 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -50,6 +50,7 @@ DEVICE_MODEL = 'model' DEVICE_MAC_ADDR = 'mac_addr' DEVICE_TEST_MODULES = 'test_modules' +DEVICE_MAX_TESTS = 'max_device_tests' class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. @@ -180,11 +181,15 @@ def _load_devices(self, device_dir): device_model = device_config_json.get(DEVICE_MODEL) mac_addr = device_config_json.get(DEVICE_MAC_ADDR) test_modules = device_config_json.get(DEVICE_TEST_MODULES) + max_device_tests = None + if 'max_device_tests' in device_config_json: + max_device_tests = device_config_json.get(DEVICE_MAX_TESTS) device = Device(manufacturer=device_manufacturer, model=device_model, mac_addr=mac_addr, - test_modules=json.dumps(test_modules)) + test_modules=json.dumps(test_modules), + max_device_tests=max_device_tests) self._devices.append(device) def get_device(self, mac_addr): diff --git a/resources/devices/template/device_config.json b/resources/devices/template/device_config.json index 1e92de25d..f6570a570 100644 --- a/resources/devices/template/device_config.json +++ b/resources/devices/template/device_config.json @@ -2,6 +2,7 @@ "manufacturer": "Manufacturer X", "model": "Device X", "mac_addr": "aa:bb:cc:dd:ee:ff", + "max_device_tests":5, "test_modules": { "dns": { "enabled": true, From c8ff7130e6c606ebfdc73005845a2db3d5cc2f13 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 26 Jul 2023 15:41:48 -0600 Subject: [PATCH 5/9] Change max report naming Add optional default value to system.json --- .gitignore | 3 +- framework/python/src/core/device.py | 2 +- framework/python/src/core/testrun.py | 12 ++--- .../python/src/test_orc/test_orchestrator.py | 45 ++++++++++++++----- local/.gitignore | 3 +- local/system.json.example | 3 +- 6 files changed, 48 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index e168ec07a..5a216522f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ venv/ error pylint.out __pycache__/ -build/ \ No newline at end of file +build/ +testing/unit_test/temp \ No newline at end of file diff --git a/framework/python/src/core/device.py b/framework/python/src/core/device.py index 824517cd0..31dfdc4a3 100644 --- a/framework/python/src/core/device.py +++ b/framework/python/src/core/device.py @@ -25,4 +25,4 @@ class Device(NetworkDevice): manufacturer: str = None model: str = None test_modules: str = None - max_device_tests: int = None + max_device_reports: int = None diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 25b20f2d3..741cbc280 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -50,7 +50,7 @@ DEVICE_MODEL = 'model' DEVICE_MAC_ADDR = 'mac_addr' DEVICE_TEST_MODULES = 'test_modules' -DEVICE_MAX_TESTS = 'max_device_tests' +MAX_DEVICE_REPORTS_KEY = 'max_device_reports' class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. @@ -79,7 +79,7 @@ def __init__(self, validate=validate, single_intf = self._single_intf) - self._test_orc = test_orc.TestOrchestrator(self._net_orc) + self._test_orc = test_orc.TestOrchestrator(self._net_orc, config_file=config_file) def start(self): @@ -181,15 +181,15 @@ def _load_devices(self, device_dir): device_model = device_config_json.get(DEVICE_MODEL) mac_addr = device_config_json.get(DEVICE_MAC_ADDR) test_modules = device_config_json.get(DEVICE_TEST_MODULES) - max_device_tests = None - if 'max_device_tests' in device_config_json: - max_device_tests = device_config_json.get(DEVICE_MAX_TESTS) + max_device_reports = None + if 'max_device_reports' in device_config_json: + max_device_reports = device_config_json.get(MAX_DEVICE_REPORTS_KEY) device = Device(manufacturer=device_manufacturer, model=device_model, mac_addr=mac_addr, test_modules=json.dumps(test_modules), - max_device_tests=max_device_tests) + max_device_reports=max_device_reports) self._devices.append(device) def get_device(self, mac_addr): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 3f60dda0b..57967bfb3 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -28,18 +28,19 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" - - +CONFIG_FILE = 'local/system.json' +MAX_DEVICE_REPORTS_KEY = "max_device_reports" +DEFAULT_MAX_DEVICE_REPORTS = 5 + class TestOrchestrator: """Manages and controls the test modules.""" - def __init__(self, net_orc): + def __init__(self, net_orc, config_file=CONFIG_FILE): self._test_modules = [] self._module_config = None self._net_orc = net_orc self._test_in_progress = False - self._max_device_tests = 5 - + self._max_device_reports = DEFAULT_MAX_DEVICE_REPORTS self._path = os.path.dirname(os.path.dirname( os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) @@ -48,6 +49,7 @@ def __init__(self, net_orc): os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) + self.load_config(config_file) def start(self): LOGGER.debug("Starting test orchestrator") @@ -56,6 +58,7 @@ def start(self): os.makedirs(RUNTIME_DIR, exist_ok=True) util.run_command(f"chown -R {self._host_user} {RUNTIME_DIR}") + self._load_test_modules() self.build_test_modules() @@ -80,16 +83,16 @@ def run_test_modules(self, device): self._test_in_progress = False def _cleanup_old_test_results(self, device): - if device.max_device_tests is not None: - max_tests = device.max_device_tests + if device.max_device_reports is not None: + max_device_reports = device.max_device_reports else: - max_tests = self._max_device_tests + max_device_reports = self._max_device_reports completed_results_dir = cur_results_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/completed_tests/") completed_tests = os.listdir(completed_results_dir) cur_test_count = len(completed_tests) - if cur_test_count > max_tests: + if cur_test_count > max_device_reports: LOGGER.debug("Current device has more than max tests results allowed: " + str(cur_test_count) + ">" + str(max_tests)) # Find and delete the oldest test oldest_test = self._find_oldest_test(completed_results_dir) @@ -98,7 +101,7 @@ def _cleanup_old_test_results(self, device): shutil.rmtree(oldest_test,ignore_errors=True) # Confirm the delete was succesful new_test_count = len(os.listdir(completed_results_dir)) - if new_test_count != cur_test_count and new_test_count > max_tests: + if new_test_count != cur_test_count and new_test_count > max_device_reports: # Continue cleaning up until we're under the max self._cleanup_old_test_results(device) @@ -283,6 +286,28 @@ def _get_module_container(self, module): LOGGER.error(error) return container + def import_config(self, json_config): + if MAX_DEVICE_REPORTS_KEY in json_config: + self._max_device_reports = json_config[MAX_DEVICE_REPORTS_KEY] + + def load_config(self, config_file=None): + if config_file is None: + # If not defined, use relative pathing to local file + self._config_file = os.path.join(self._path, CONFIG_FILE) + else: + # If defined, use as provided + self._config_file = config_file + + if not os.path.isfile(self._config_file): + LOGGER.error('Configuration file is not present at ' + config_file) + LOGGER.info('An example is present in ' + EXAMPLE_CONFIG_FILE) + sys.exit(1) + + LOGGER.info('Loading config file: ' + os.path.abspath(self._config_file)) + with open(self._config_file, encoding='UTF-8') as config_json_file: + config_json = json.load(config_json_file) + self.import_config(config_json) + def _load_test_modules(self): """Load network modules from module_config.json.""" LOGGER.debug("Loading test modules from /" + TEST_MODULES_DIR) diff --git a/local/.gitignore b/local/.gitignore index 4fb365c03..f13ce8d85 100644 --- a/local/.gitignore +++ b/local/.gitignore @@ -1,2 +1,3 @@ system.json -devices \ No newline at end of file +devices +root_certs \ No newline at end of file diff --git a/local/system.json.example b/local/system.json.example index e99e013f3..17e5b0891 100644 --- a/local/system.json.example +++ b/local/system.json.example @@ -6,5 +6,6 @@ "log_level": "INFO", "startup_timeout": 60, "monitor_period": 300, - "runtime": 1200 + "runtime": 1200, + "max_device_reports": 5 } \ No newline at end of file From e22703cf465aeb71b7aceca8501d9ace53173b6e Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 26 Jul 2023 15:56:21 -0600 Subject: [PATCH 6/9] Copy current test instead of moving to keep a consistent location of the most recent test --- framework/python/src/test_orc/test_orchestrator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 57967bfb3..038457575 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -118,7 +118,6 @@ def _find_oldest_test(self,completed_tests_dir): else: return None - def _timestamp_results(self, device): # Define the current device results directory cur_results_dir = os.path.join( @@ -129,9 +128,10 @@ def _timestamp_results(self, device): completed_results_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/completed_tests/" + cur_time) - # Move the results to the timestamp directory - shutil.move(cur_results_dir,completed_results_dir) - + # Copy the results to the timestamp directory + # leave current copy in place for quick reference to + # most recent test + shutil.copytree(cur_results_dir,completed_results_dir) def _generate_results(self, device): results = {} From 14133006b0d2baef03fcc4dca27ce0c273562502 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 26 Jul 2023 16:40:31 -0600 Subject: [PATCH 7/9] fix merge issue --- framework/python/src/core/testrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 5cf2ae970..36bec1d09 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -131,7 +131,7 @@ def _load_devices(self, device_dir): mac_addr=mac_addr, test_modules=test_modules, max_device_reports=max_device_reports) - self._devices.append(device) + self.get_session().add_device(device) self.get_session().add_device(device) LOGGER.debug(f'Loaded device {device.manufacturer} {device.model} with MAC address {device.mac_addr}') From ff1515d1281e54044eba690e9ee43af65241b159 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 26 Jul 2023 17:09:41 -0600 Subject: [PATCH 8/9] pylint --- framework/python/src/common/device.py | 2 +- .../python/src/test_orc/test_orchestrator.py | 90 ++++++++++--------- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index f75a133d7..b06047d4a 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -25,4 +25,4 @@ class Device(): model: str = None test_modules: str = None ip_addr: str = None - max_device_reports: int = None \ No newline at end of file + max_device_reports: int = None diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index b82c26ac9..4d1ff1601 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Provides high level management of the test orchestrator.""" import os import json import time import shutil import docker +import sys from datetime import datetime from docker.types import Mount from common import logger, util @@ -28,10 +28,12 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" -CONFIG_FILE = 'local/system.json' +CONFIG_FILE = "local/system.json" +EXAMPLE_CONFIG_FILE = "local/system.json.example" MAX_DEVICE_REPORTS_KEY = "max_device_reports" DEFAULT_MAX_DEVICE_REPORTS = 5 - + + class TestOrchestrator: """Manages and controls the test modules.""" @@ -42,15 +44,18 @@ def __init__(self, session, net_orc, config_file=CONFIG_FILE): self._net_orc = net_orc self._test_in_progress = False self._max_device_reports = DEFAULT_MAX_DEVICE_REPORTS - self._path = os.path.dirname(os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) + self._path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - self._root_path = os.path.dirname(os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) + self._root_path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) self.load_config(config_file) + def start(self): LOGGER.debug("Starting test orchestrator") @@ -59,7 +64,6 @@ def start(self): os.makedirs(RUNTIME_DIR, exist_ok=True) util.run_command(f"chown -R {self._host_user} {RUNTIME_DIR}") - self._load_test_modules() self.build_test_modules() @@ -88,51 +92,53 @@ def _cleanup_old_test_results(self, device): max_device_reports = device.max_device_reports else: max_device_reports = self._max_device_reports - completed_results_dir = cur_results_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/completed_tests/") + completed_results_dir = os.path.join( + self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + + "/completed_tests/") completed_tests = os.listdir(completed_results_dir) cur_test_count = len(completed_tests) if cur_test_count > max_device_reports: - LOGGER.debug("Current device has more than max tests results allowed: " + str(cur_test_count) + ">" + str(max_tests)) + LOGGER.debug("Current device has more than max tests results allowed: " + + str(cur_test_count) + ">" + str(max_device_reports)) # Find and delete the oldest test oldest_test = self._find_oldest_test(completed_results_dir) if oldest_test is not None: LOGGER.debug("Oldest test found, removing: " + str(oldest_test)) - shutil.rmtree(oldest_test,ignore_errors=True) + shutil.rmtree(oldest_test, ignore_errors=True) # Confirm the delete was succesful new_test_count = len(os.listdir(completed_results_dir)) - if new_test_count != cur_test_count and new_test_count > max_device_reports: + if (new_test_count != cur_test_count + and new_test_count > max_device_reports): # Continue cleaning up until we're under the max self._cleanup_old_test_results(device) - - def _find_oldest_test(self,completed_tests_dir): + + def _find_oldest_test(self, completed_tests_dir): oldest_timestamp = None oldest_directory = None for completed_test in os.listdir(completed_tests_dir): - timestamp = datetime.strptime(str(completed_test), '%Y-%m-%dT%H:%M:%S') - if oldest_timestamp is None or timestamp < oldest_timestamp: - oldest_timestamp = timestamp - oldest_directory = completed_test + timestamp = datetime.strptime(str(completed_test), "%Y-%m-%dT%H:%M:%S") + if oldest_timestamp is None or timestamp < oldest_timestamp: + oldest_timestamp = timestamp + oldest_directory = completed_test if oldest_directory: - return os.path.join(completed_tests_dir, oldest_directory) + return os.path.join(completed_tests_dir, oldest_directory) else: - return None + return None def _timestamp_results(self, device): # Define the current device results directory cur_results_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/current_test") + self._root_path, + "runtime/test/" + device.mac_addr.replace(":", "") + "/current_test") # Define the destination results directory with timestamp - cur_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + cur_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") completed_results_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/completed_tests/" + cur_time) + self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + + "/completed_tests/" + cur_time) # Copy the results to the timestamp directory # leave current copy in place for quick reference to # most recent test - shutil.copytree(cur_results_dir,completed_results_dir) + shutil.copytree(cur_results_dir, completed_results_dir) def _generate_results(self, device): results = {} @@ -154,12 +160,15 @@ def _generate_results(self, device): results[module.name] = module_results except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: - LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") + LOGGER.error( + f("Error occured whilst obbtaining results " + "for module {module.name}") + ) LOGGER.debug(results_error) out_file = os.path.join( - self._root_path, - "runtime/test/" + device.mac_addr.replace(":", "") + "/current_test/results.json") + self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + + "/current_test/results.json") with open(out_file, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") @@ -256,13 +265,12 @@ def _run_test_module(self, module, device): status = self._get_module_status(module) log_stream = module.container.logs(stream=True, stdout=True, stderr=True) - while (time.time() < test_module_timeout and - status == "running" and - self._session.get_status() == "In progress"): + while (time.time() < test_module_timeout and status == "running" + and self._session.get_status() == "In progress"): try: line = next(log_stream).decode("utf-8").strip() print(line) - except Exception: + except Exception: # pylint: disable=W0718 time.sleep(1) status = self._get_module_status(module) @@ -307,12 +315,12 @@ def load_config(self, config_file=None): self._config_file = config_file if not os.path.isfile(self._config_file): - LOGGER.error('Configuration file is not present at ' + config_file) - LOGGER.info('An example is present in ' + EXAMPLE_CONFIG_FILE) + LOGGER.error("Configuration file is not present at " + config_file) + LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) sys.exit(1) - LOGGER.info('Loading config file: ' + os.path.abspath(self._config_file)) - with open(self._config_file, encoding='UTF-8') as config_json_file: + LOGGER.info("Loading config file: " + os.path.abspath(self._config_file)) + with open(self._config_file, encoding="UTF-8") as config_json_file: config_json = json.load(config_json_file) self.import_config(config_json) From 9072f97374e93569792e6b9202525324d8d61f28 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 3 Aug 2023 16:38:48 +0100 Subject: [PATCH 9/9] Use local device folder and use session for config --- framework/python/src/common/device.py | 1 + framework/python/src/common/session.py | 10 +- framework/python/src/core/device.py | 28 ---- framework/python/src/core/testrun.py | 16 ++- .../src/net_orc/network_orchestrator.py | 31 +++-- .../python/src/test_orc/test_orchestrator.py | 126 ++++++++++-------- 6 files changed, 112 insertions(+), 100 deletions(-) delete mode 100644 framework/python/src/core/device.py diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index b06047d4a..83f0c1a15 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -25,4 +25,5 @@ class Device(): model: str = None test_modules: str = None ip_addr: str = None + device_folder: str = None max_device_reports: int = None diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index a0f6118ff..7306d5cc8 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -25,6 +25,7 @@ MONITOR_PERIOD_KEY = 'monitor_period' STARTUP_TIMEOUT_KEY = 'startup_timeout' LOG_LEVEL_KEY = 'log_level' +MAX_DEVICE_REPORTS_KEY = 'max_device_reports' class TestRunSession(): """Represents the current session of Test Run.""" @@ -62,7 +63,8 @@ def _get_default_config(self): 'log_level': 'INFO', 'startup_timeout': 60, 'monitor_period': 30, - 'runtime': 120 + 'runtime': 120, + 'max_device_reports': 5 } def get_config(self): @@ -95,6 +97,9 @@ def _load_config(self): if LOG_LEVEL_KEY in config_file_json: self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY) + if MAX_DEVICE_REPORTS_KEY in config_file_json: + self._config[MAX_DEVICE_REPORTS_KEY] = config_file_json.get(MAX_DEVICE_REPORTS_KEY) + def _save_config(self): with open(self._config_file, 'w', encoding='utf-8') as f: f.write(json.dumps(self._config, indent=2)) @@ -116,6 +121,9 @@ def get_monitor_period(self): def get_startup_timeout(self): return self._config.get(STARTUP_TIMEOUT_KEY) + + def get_max_device_reports(self): + return self._config.get(MAX_DEVICE_REPORTS_KEY) def set_config(self, config_json): self._config = config_json diff --git a/framework/python/src/core/device.py b/framework/python/src/core/device.py deleted file mode 100644 index 31dfdc4a3..000000000 --- a/framework/python/src/core/device.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Track device object information.""" - -from net_orc.network_device import NetworkDevice -from dataclasses import dataclass - - -@dataclass -class Device(NetworkDevice): - """Represents a physical device and it's configuration.""" - - manufacturer: str = None - model: str = None - test_modules: str = None - max_device_reports: int = None diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 36bec1d09..0c3de6db4 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -90,8 +90,7 @@ def __init__(self, single_intf = self._single_intf) self._test_orc = test_orc.TestOrchestrator( self._session, - self._net_orc, - config_file=config_file) + self._net_orc) if self._no_ui: self.start() @@ -114,7 +113,15 @@ def _load_devices(self, device_dir): util.run_command(f'chown -R {util.get_host_user()} {device_dir}') for device_folder in os.listdir(device_dir): - with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), + + device_config_file_path = os.path.join(device_dir, + device_folder, + DEVICE_CONFIG) + if not os.path.exists(device_config_file_path): + LOGGER.error(f'Device configuration file missing from device {device_folder}') + continue + + with open(device_config_file_path, encoding='utf-8') as device_config_file: device_config_json = json.load(device_config_file) @@ -130,7 +137,8 @@ def _load_devices(self, device_dir): model=device_model, mac_addr=mac_addr, test_modules=test_modules, - max_device_reports=max_device_reports) + max_device_reports=max_device_reports, + device_folder=device_folder) self.get_session().add_device(device) self.get_session().add_device(device) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 283814f53..4c56a05f0 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -154,21 +154,25 @@ def _device_discovered(self, mac_addr): # Ignore device if not registered return - device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, - mac_addr.replace(':', '') + '/current_test') - + device_runtime_dir = os.path.join(RUNTIME_DIR, + TEST_DIR, + mac_addr.replace(':', '') + ) + # Cleanup any old current test files shutil.rmtree(device_runtime_dir, ignore_errors=True) os.makedirs(device_runtime_dir, exist_ok=True) util.run_command(f'chown -R {self._host_user} {device_runtime_dir}') - + packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_startup_timeout(), stop_filter=self._device_has_ip) wrpcap( - os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), - 'current_test/startup.pcap'), packet_capture) + os.path.join(device_runtime_dir, + 'startup.pcap' + ), + packet_capture) if device.ip_addr is None: LOGGER.info( @@ -205,14 +209,23 @@ def _start_device_monitor(self, device): callback the steady state method for this device.""" LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' f'for {str(self._session.get_monitor_period())} seconds') + + device_runtime_dir = os.path.join(RUNTIME_DIR, + TEST_DIR, + device.mac_addr.replace(':', '') + ) packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_monitor_period()) wrpcap( - os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), - 'current_test/monitor.pcap'), packet_capture) + os.path.join(device_runtime_dir, + 'monitor.pcap' + ), + packet_capture) self._monitor_in_progress = False - self.get_listener().call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) + self.get_listener().call_callback( + NetworkEvent.DEVICE_STABLE, + device.mac_addr) def _check_network_services(self): LOGGER.debug('Checking network modules...') diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 4d1ff1601..afae0ce67 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -17,7 +17,6 @@ import time import shutil import docker -import sys from datetime import datetime from docker.types import Mount from common import logger, util @@ -28,22 +27,17 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" -CONFIG_FILE = "local/system.json" -EXAMPLE_CONFIG_FILE = "local/system.json.example" -MAX_DEVICE_REPORTS_KEY = "max_device_reports" -DEFAULT_MAX_DEVICE_REPORTS = 5 +SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports" class TestOrchestrator: """Manages and controls the test modules.""" - def __init__(self, session, net_orc, config_file=CONFIG_FILE): + def __init__(self, session, net_orc): self._test_modules = [] - self._module_config = None self._session = session self._net_orc = net_orc self._test_in_progress = False - self._max_device_reports = DEFAULT_MAX_DEVICE_REPORTS self._path = os.path.dirname( os.path.dirname( os.path.dirname( @@ -54,8 +48,6 @@ def __init__(self, session, net_orc, config_file=CONFIG_FILE): os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - self.load_config(config_file) - def start(self): LOGGER.debug("Starting test orchestrator") @@ -82,24 +74,30 @@ def run_test_modules(self, device): self._generate_results(device) self._timestamp_results(device) - LOGGER.info("Cleaning old test results...") + LOGGER.debug("Cleaning old test results...") self._cleanup_old_test_results(device) - LOGGER.info("Old test results cleaned") + LOGGER.debug("Old test results cleaned") self._test_in_progress = False def _cleanup_old_test_results(self, device): + if device.max_device_reports is not None: max_device_reports = device.max_device_reports else: - max_device_reports = self._max_device_reports + max_device_reports = self._session.get_max_device_reports() + completed_results_dir = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/completed_tests/") + self._root_path, + SAVED_DEVICE_REPORTS.replace("{device_folder}", + device.device_folder) + ) + completed_tests = os.listdir(completed_results_dir) cur_test_count = len(completed_tests) if cur_test_count > max_device_reports: LOGGER.debug("Current device has more than max tests results allowed: " + str(cur_test_count) + ">" + str(max_device_reports)) + # Find and delete the oldest test oldest_test = self._find_oldest_test(completed_results_dir) if oldest_test is not None: @@ -126,38 +124,56 @@ def _find_oldest_test(self, completed_tests_dir): return None def _timestamp_results(self, device): + # Define the current device results directory cur_results_dir = os.path.join( self._root_path, - "runtime/test/" + device.mac_addr.replace(":", "") + "/current_test") + RUNTIME_DIR, + device.mac_addr.replace(":", "") + ) + # Define the destination results directory with timestamp cur_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") completed_results_dir = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/completed_tests/" + cur_time) + SAVED_DEVICE_REPORTS.replace("{device_folder}", + device.device_folder), + cur_time) + # Copy the results to the timestamp directory # leave current copy in place for quick reference to # most recent test shutil.copytree(cur_results_dir, completed_results_dir) + util.run_command(f"chown -R {self._host_user} '{completed_results_dir}'") def _generate_results(self, device): - results = {} - results["device"] = {} + + report = {} + + report["device"] = {} if device.manufacturer is not None: - results["device"]["manufacturer"] = device.manufacturer + report["device"]["manufacturer"] = device.manufacturer if device.model is not None: - results["device"]["model"] = device.model - results["device"]["mac_addr"] = device.mac_addr + report["device"]["model"] = device.model + report["device"]["mac_addr"] = device.mac_addr + + results = [] + for module in self._test_modules: if module.enable_container and self._is_module_enabled(module, device): + container_runtime_dir = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + "/current_test/" + module.name) + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", ""), + module.name + ) + results_file = f"{container_runtime_dir}/{module.name}-result.json" try: with open(results_file, "r", encoding="utf-8-sig") as f: module_results = json.load(f) - results[module.name] = module_results + for result in module_results["results"]: + results.append(result) except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: LOGGER.error( @@ -166,13 +182,19 @@ def _generate_results(self, device): ) LOGGER.debug(results_error) + report["results"] = results + out_file = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/current_test/results.json") + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", ""), + "report.json" + ) + with open(out_file, "w", encoding="utf-8") as f: - json.dump(results, f, indent=2) + json.dump(report, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") - return results + return report def test_in_progress(self): return self._test_in_progress @@ -198,21 +220,31 @@ def _run_test_module(self, module, device): LOGGER.info("Running test module " + module.name) try: + + device_test_dir = os.path.join( + self._root_path, + RUNTIME_DIR, + device.mac_addr.replace(":", "") + ) + container_runtime_dir = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/current_test/" + module.name) + device_test_dir, + module.name + ) os.makedirs(container_runtime_dir, exist_ok=True) network_runtime_dir = os.path.join(self._root_path, "runtime/network") device_startup_capture = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/current_test/startup.pcap") + device_test_dir, + "startup.pcap" + ) util.run_command(f"chown -R {self._host_user} {device_startup_capture}") device_monitor_capture = os.path.join( - self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + - "/current_test/monitor.pcap") + device_test_dir, + "monitor.pcap" + ) util.run_command(f"chown -R {self._host_user} {device_monitor_capture}") client = docker.from_env() @@ -302,28 +334,6 @@ def _get_module_container(self, module): LOGGER.error(error) return container - def import_config(self, json_config): - if MAX_DEVICE_REPORTS_KEY in json_config: - self._max_device_reports = json_config[MAX_DEVICE_REPORTS_KEY] - - def load_config(self, config_file=None): - if config_file is None: - # If not defined, use relative pathing to local file - self._config_file = os.path.join(self._path, CONFIG_FILE) - else: - # If defined, use as provided - self._config_file = config_file - - if not os.path.isfile(self._config_file): - LOGGER.error("Configuration file is not present at " + config_file) - LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) - sys.exit(1) - - LOGGER.info("Loading config file: " + os.path.abspath(self._config_file)) - with open(self._config_file, encoding="UTF-8") as config_json_file: - config_json = json.load(config_json_file) - self.import_config(config_json) - def _load_test_modules(self): """Load network modules from module_config.json.""" LOGGER.debug("Loading test modules from /" + TEST_MODULES_DIR)