diff --git a/.gitignore b/.gitignore index 4016b6901..f79a6efcb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,135 +1,135 @@ -# Runtime folder -runtime/ -venv/ -net_orc/ -.vscode/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +# Runtime folder +runtime/ +venv/ +net_orc/ +.vscode/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cmd/install b/cmd/install index 61722e273..6dee1c635 100755 --- a/cmd/install +++ b/cmd/install @@ -2,7 +2,7 @@ GIT_URL=https://github.com/auto-iot NET_ORC_DIR=net_orc -NET_ORC_VERSION="dev" +NET_ORC_VERSION="main" python3 -m venv venv diff --git a/cmd/start b/cmd/start index fa6bbc1e1..113f14b3e 100755 --- a/cmd/start +++ b/cmd/start @@ -18,6 +18,6 @@ rm -rf runtime source venv/bin/activate # TODO: Execute python code -python -u framework/run.py $@ +python -u framework/test_runner.py $@ deactivate \ No newline at end of file diff --git a/framework/run.py b/framework/run.py deleted file mode 100644 index fc6c197e3..000000000 --- a/framework/run.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Starts Test Run.""" - -import argparse -import sys -from testrun import TestRun -import logger - -LOGGER = logger.get_logger('runner') - -class TestRunner: - - def __init__(self, local_net=True): - - LOGGER.info('Starting Test Run') - - testrun = TestRun(local_net) - - testrun.load_config() - - testrun.start() - - testrun.run_tests() - - testrun.stop_network() - - -def run(argv): - parser = argparse.ArgumentParser(description="Test Run", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-r", "--remote-net", action="store_false", - help='''Use the network orchestrator from the parent directory instead - of the one downloaded locally from the install script.''') - - args, unknown = parser.parse_known_args() - - TestRunner(args.remote_net) - - -if __name__ == "__main__": - run(sys.argv) diff --git a/framework/test_runner.py b/framework/test_runner.py new file mode 100644 index 000000000..91ff4cb1a --- /dev/null +++ b/framework/test_runner.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +"""Wrapper for the TestRun that simplifies +virtual testing procedure by allowing direct calling +from the command line. + +Run using the provided command scripts in the cmd folder. +E.g sudo cmd/start +""" + +import argparse +import sys +from testrun import TestRun +import logger +import signal + +LOGGER = logger.get_logger('runner') + + +class TestRunner: + + def __init__(self, local_net=True, config_file=None, validate=True, net_only=False): + self._register_exits() + self.test_run = TestRun(local_net=local_net, config_file=config_file, + validate=validate, net_only=net_only) + + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) + + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received.") + # Kill all container services quickly + # If we're here, we want everything to stop immediately + # and don't care about a gracefully shutdown + self._stop(True) + sys.exit(1) + + def stop(self, kill=False): + self.test_run.stop(kill) + + def start(self): + self.test_run.start() + LOGGER.info("Test Run has finished") + + +def parse_args(argv): + parser = argparse.ArgumentParser(description="Test Run", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-r", "--remote-net", action="store_false", + help='''Use the network orchestrator from the parent directory instead + of the one downloaded locally from the install script.''') + parser.add_argument("-f", "--config-file", default=None, + help="Define the configuration file for Test Run and Network Orchestrator") + parser.add_argument("--no-validate", action="store_true", + help="Turn off the validation of the network after network boot") + parser.add_argument("-net", "--net-only", action="store_true", + help="Run the network only, do not run tests") + args, unknown = parser.parse_known_args() + return args + + +if __name__ == "__main__": + args = parse_args(sys.argv) + runner = TestRunner(local_net=args.remote_net, + config_file=args.config_file, + validate=not args.no_validate, + net_only=args.net_only) + runner.start() diff --git a/framework/testrun.py b/framework/testrun.py index 372a64692..4a29b4e20 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -19,8 +19,8 @@ parent_dir = os.path.dirname(current_dir) LOGGER = logger.get_logger('test_run') -CONFIG_FILE = 'conf/system.json' -EXAMPLE_CONFIG_FILE = 'conf/system.json.example' +CONFIG_FILE = "conf/system.json" +EXAMPLE_CONFIG_FILE = "conf/system.json.example" RUNTIME = 300 DEVICES_DIR = 'local/devices' @@ -31,150 +31,142 @@ class TestRun: # pylint: disable=too-few-public-methods - """Test Run controller. - - Creates an instance of the network orchestrator, test - orchestrator and user interface. - """ - - def __init__(self, local_net=True): - self._devices = [] - - # Catch any exit signals - self._register_exits() - - # Import the correct net orchestrator - self.import_dependencies(local_net) - - self._net_orc = net_orc.NetworkOrchestrator() - self._test_orc = test_orc.TestOrchestrator() - - def start(self): - - self._load_devices() - - self.start_network() - - # Register callbacks - self._net_orc.listener.register_callback( - self._device_discovered, - [NetworkEvent.DEVICE_DISCOVERED]) - - def import_dependencies(self, local_net=True): - """Imports both net and test orchestrators from relevant directories.""" - if local_net: - # Add local net_orc to Python path - net_orc_dir = os.path.join( - parent_dir, 'net_orc', 'python', 'src') - else: - # Resolve the path to the test-run parent folder - root_dir = os.path.abspath(os.path.join(parent_dir, os.pardir)) - # Add manually cloned network orchestrator from parent folder - net_orc_dir = os.path.join( - root_dir, 'network-orchestrator', 'python', 'src') - # Add net_orc to Python path - sys.path.append(net_orc_dir) - # Import the network orchestrator - global net_orc - import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel - - # Add test_orc to Python path - test_orc_dir = os.path.join( - parent_dir, 'test_orc', 'python', 'src') - sys.path.append(test_orc_dir) - global test_orc - import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel - - global NetworkEvent - from listener import NetworkEvent # pylint: disable=wrong-import-position,import-outside-toplevel - - def _register_exits(self): - signal.signal(signal.SIGINT, self._exit_handler) - signal.signal(signal.SIGTERM, self._exit_handler) - signal.signal(signal.SIGABRT, self._exit_handler) - signal.signal(signal.SIGQUIT, self._exit_handler) - - def _exit_handler(self, signum, arg): # pylint: disable=unused-argument - LOGGER.debug('Exit signal received: ' + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info('Exit signal received.') - self.stop_network() - - def load_config(self): - """Loads all settings from the config file into memory.""" - if not os.path.isfile(CONFIG_FILE): - LOGGER.error( - 'Configuration file is not present at ' + CONFIG_FILE) - LOGGER.info('An example is present in ' + EXAMPLE_CONFIG_FILE) - sys.exit(1) - - with open(CONFIG_FILE, 'r', encoding='UTF-8') as config_file_open: - config_json = json.load(config_file_open) - self._net_orc.import_config(config_json) - self._test_orc.import_config(config_json) - - def start_network(self): - """Starts the network orchestrator and network services.""" - - # Load and build any unbuilt network containers - self._net_orc.load_network_modules() - self._net_orc.build_network_modules() - - self._net_orc.stop_networking_services(kill=True) - self._net_orc.restore_net() - - # Create baseline network - self._net_orc.create_net() - - # Launch network service containers - self._net_orc.start_network_services() - - LOGGER.info('Network is ready.') - - def run_tests(self): - """Iterate through and start all test modules.""" - self._test_orc.load_test_modules() - self._test_orc.build_test_modules() - - # Begin testing - self._test_orc.run_test_modules() - - def stop_network(self): - """Commands the net_orc to stop the network and clean up.""" - self._net_orc.stop_networking_services(kill=True) - self._net_orc.restore_net() - sys.exit(0) - - def _load_devices(self): - LOGGER.debug('Loading devices from ' + DEVICES_DIR) - - for device_folder in os.listdir(DEVICES_DIR): - with open(os.path.join(DEVICES_DIR, device_folder, DEVICE_CONFIG), - encoding='utf-8') as device_config_file: - device_config_json = json.load(device_config_file) - - device_make = device_config_json.get(DEVICE_MAKE) - device_model = device_config_json.get(DEVICE_MODEL) - mac_addr = device_config_json.get(DEVICE_MAC_ADDR) - - device = Device(device_make, device_model, - mac_addr=mac_addr) - self._devices.append(device) - - LOGGER.info('Loaded ' + str(len(self._devices)) + ' devices') - - def get_device(self, mac_addr): - """Returns a loaded device object from the device mac address.""" - for device in self._devices: - if device.mac_addr == mac_addr: + """Test Run controller. + + Creates an instance of the network orchestrator, test + orchestrator and user interface. + """ + + def __init__(self, local_net=True, config_file=CONFIG_FILE,validate=True, net_only=False): + self._devices = [] + self._net_only = net_only + + # Catch any exit signals + self._register_exits() + + # Import the correct net orchestrator + self.import_dependencies(local_net) + + # Expand the config file to absolute pathing + config_file_abs=self._get_config_abs(config_file=config_file) + + self._net_orc = net_orc.NetworkOrchestrator(config_file=config_file_abs,validate=validate,async_monitor=not self._net_only) + self._test_orc = test_orc.TestOrchestrator() + + def start(self): + + self._load_devices() + + if self._net_only: + LOGGER.info("Network only option configured, no tests will be run") + self._start_network() + else: + self._start_network() + self._start_tests() + + self.stop() + + # Register callbacks + # Disable for now as this is causing boot failures when no devices are discovered + # self._net_orc.listener.register_callback( + # self._device_discovered, + # [NetworkEvent.DEVICE_DISCOVERED]) + + def stop(self,kill=False): + self._stop_tests() + self._stop_network(kill=kill) + + def import_dependencies(self, local_net=True): + if local_net: + # Add local net_orc to Python path + net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') + else: + # Resolve the path to the test-run parent folder + root_dir = os.path.abspath(os.path.join(parent_dir, os.pardir)) + # Add manually cloned network orchestrator from parent folder + net_orc_dir = os.path.join( + root_dir, 'network-orchestrator', 'python', 'src') + # Add net_orc to Python path + sys.path.append(net_orc_dir) + # Import the network orchestrator + global net_orc + import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel + + # Add test_orc to Python path + test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') + sys.path.append(test_orc_dir) + global test_orc + import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel + + global NetworkEvent + from listener import NetworkEvent # pylint: disable=wrong-import-position,import-outside-toplevel + + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) + + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received.") + self.stop(kill=True) + sys.exit(1) + + def _get_config_abs(self,config_file=None): + if config_file is None: + # If not defined, use relative pathing to local file + config_file = os.path.join(parent_dir, CONFIG_FILE) + + # Expand the config file to absolute pathing + return os.path.abspath(config_file) + + def _start_network(self): + self._net_orc.start() + + def _start_tests(self): + """Iterate through and start all test modules.""" + + self._test_orc.start() + + def _stop_network(self,kill=False): + self._net_orc.stop(kill=kill) + + def _stop_tests(self): + self._test_orc.stop() + + def _load_devices(self): + LOGGER.debug('Loading devices from ' + DEVICES_DIR) + + for device_folder in os.listdir(DEVICES_DIR): + with open(os.path.join(DEVICES_DIR, device_folder, DEVICE_CONFIG), + encoding='utf-8') as device_config_file: + device_config_json = json.load(device_config_file) + + device_make = device_config_json.get(DEVICE_MAKE) + device_model = device_config_json.get(DEVICE_MODEL) + mac_addr = device_config_json.get(DEVICE_MAC_ADDR) + + device = Device(device_make, device_model, + mac_addr=mac_addr) + self._devices.append(device) + + LOGGER.info('Loaded ' + str(len(self._devices)) + ' devices') + + def get_device(self, mac_addr): + """Returns a loaded device object from the device mac address.""" + for device in self._devices: + if device.mac_addr == mac_addr: + return device + return None + + def _device_discovered(self, mac_addr): + device = self.get_device(mac_addr) + if device is not None: + LOGGER.info( + f'Discovered {device.make} {device.model} on the network') + else: + LOGGER.info( + f'A new device has been discovered with mac address {device.mac_addr}') return device - return None - - def _device_discovered(self, mac_addr): - device = self.get_device(mac_addr) - if device is not None: - LOGGER.info( - f'Discovered {device.make} {device.model} on the network') - else: - LOGGER.info( - f'A new device has been discovered with mac address {device.mac_addr}') diff --git a/test_orc/python/src/module.py b/test_orc/python/src/module.py new file mode 100644 index 000000000..6d24d7e1e --- /dev/null +++ b/test_orc/python/src/module.py @@ -0,0 +1,23 @@ +"""Represemts a test module.""" +from dataclasses import dataclass +from docker.client.Container import Container + +@dataclass +class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes + """Represents a test module.""" + + name: str = None + display_name: str = None + description: str = None + + build_file: str = None + container: Container = None + container_name: str = None + image_name :str = None + enable_container: bool = True + + timeout: int = 60 + + # Absolute path + dir: str = None + dir_name: str = None diff --git a/test_orc/python/src/runner.py b/test_orc/python/src/runner.py new file mode 100644 index 000000000..cc495bf8d --- /dev/null +++ b/test_orc/python/src/runner.py @@ -0,0 +1,40 @@ +"""Provides high level management of the test orchestrator.""" +import time +import logger + +LOGGER = logger.get_logger('runner') + +class Runner: + """Holds the state of the testing for one device.""" + + def __init__(self, test_orc, device): + self._test_orc = test_orc + self._device = device + + def run(self): + self._run_test_modules() + + def _run_test_modules(self): + """Iterates through each test module and starts the container.""" + LOGGER.info('Running test modules...') + for module in self._test_modules: + self.run_test_module(module) + LOGGER.info('All tests complete') + + def run_test_module(self, module): + """Start the test container and extract the results.""" + + if module is None or not module.enable_container: + return + + self._test_orc.start_test_module(module) + + # Determine the module timeout time + test_module_timeout = time.time() + module.timeout + status = self._test_orc.get_module_status(module) + + while time.time() < test_module_timeout and status == 'running': + time.sleep(1) + status = self._test_orc.get_module_status(module) + + LOGGER.info(f'Test module {module.display_name} has finished') diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 396f533fa..77f73f407 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -29,32 +29,29 @@ def __init__(self): shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR), ignore_errors=True) os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) - def import_config(self, json_config): - """Load settings from JSON object into memory.""" + def start(self): + LOGGER.info("Starting Test Orchestrator") + self._load_test_modules() + self._run_test_modules() - # No relevant config options in system.json as of yet + def stop(self): + """Stop any running tests""" + self._stop_modules() - def get_test_module(self, name): - """Returns a test module by the module name.""" - for module in self._test_modules: - if name == module.name: - return module - return None - - def run_test_modules(self): + def _run_test_modules(self): """Iterates through each test module and starts the container.""" LOGGER.info("Running test modules...") for module in self._test_modules: - self.run_test_module(module) + self._run_test_module(module) LOGGER.info("All tests complete") - def run_test_module(self, module): + def _run_test_module(self, module): """Start the test container and extract the results.""" if module is None or not module.enable_container: return - LOGGER.info("Running test module " + module.display_name) + LOGGER.info("Running test module " + module.name) try: container_runtime_dir = os.path.join(self._root_path, "runtime/test/" + module.name) @@ -78,7 +75,7 @@ def run_test_module(self, module): environment={"HOST_USER": os.getlogin()} ) except (docker.errors.APIError, docker.errors.ContainerError) as container_error: - LOGGER.error("Test module " + module.display_name + " has failed to start") + LOGGER.error("Test module " + module.name + " has failed to start") LOGGER.debug(container_error) return @@ -90,7 +87,7 @@ def run_test_module(self, module): time.sleep(1) status = self._get_module_status(module) - LOGGER.info("Test module " + module.display_name + " has finished") + LOGGER.info("Test module " + module.name + " has finished") def _get_module_status(self,module): container = self._get_module_container(module) @@ -111,7 +108,7 @@ def _get_module_container(self, module): LOGGER.error(error) return container - def load_test_modules(self): + def _load_test_modules(self): """Import module configuration from module_config.json.""" modules_dir = os.path.join(self._path, TEST_MODULES_DIR) @@ -151,7 +148,8 @@ def load_test_modules(self): self._test_modules.append(module) - loaded_modules += module.dir_name + " " + if module.enable_container: + loaded_modules += module.dir_name + " " LOGGER.info(loaded_modules) @@ -175,12 +173,13 @@ def _build_test_module(self, module): LOGGER.error(error) def _stop_modules(self, kill=False): - LOGGER.debug("Stopping test modules") + LOGGER.info("Stopping test modules") for module in self._test_modules: # Test modules may just be Docker images, so we do not want to stop them if not module.enable_container: continue self._stop_module(module, kill) + LOGGER.info("All test modules have been stopped") def _stop_module(self, module, kill=False): LOGGER.debug("Stopping test module " + module.container_name) @@ -196,9 +195,8 @@ def _stop_module(self, module, kill=False): module.container_name) container.stop() LOGGER.debug("Container stopped:" + module.container_name) - except Exception as error: - LOGGER.error("Container stop error") - LOGGER.error(error) + except docker.errors.NotFound: + pass class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Represents a test module."""