From 49c32de5a5bfa9b4f44167bec15cc9782b0d97ea Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 08:06:54 -0600 Subject: [PATCH 01/20] Initial work on test-orchestrator --- .gitignore | 132 +++++++++++++- cmd/install | 3 +- conf/system.json.example | 15 +- framework/testrun.py | 15 ++ .../modules/base/base.Dockerfile | 23 +++ test-orchestrator/modules/base/bin/capture | 30 +++ .../modules/base/bin/setup_binaries | 10 + test-orchestrator/modules/base/bin/start_grpc | 17 ++ .../modules/base/bin/start_module | 79 ++++++++ .../modules/base/bin/start_network_service | 13 ++ .../modules/base/bin/wait_for_interface | 22 +++ .../modules/base/conf/module_config.json | 12 ++ .../modules/base/python/requirements.txt | 2 + .../base/python/src/grpc/start_server.py | 34 ++++ .../modules/base/python/src/logger.py | 47 +++++ .../template/bin/start_network_service | 40 ++++ .../modules/template/conf/module_config.json | 26 +++ .../modules/template/python/src/logger.py | 48 +++++ .../modules/template/python/src/run.py | 51 ++++++ .../template/python/src/test_module.py | 55 ++++++ .../modules/template/template.Dockerfile | 11 ++ test-orchestrator/python/requirements.txt | 0 .../python/src/docker_control.py | 171 ++++++++++++++++++ test-orchestrator/python/src/logger.py | 48 +++++ test-orchestrator/python/src/run.py | 77 ++++++++ .../python/src/test_orchestrator.py | 40 ++++ 26 files changed, 1014 insertions(+), 7 deletions(-) create mode 100644 test-orchestrator/modules/base/base.Dockerfile create mode 100644 test-orchestrator/modules/base/bin/capture create mode 100644 test-orchestrator/modules/base/bin/setup_binaries create mode 100644 test-orchestrator/modules/base/bin/start_grpc create mode 100644 test-orchestrator/modules/base/bin/start_module create mode 100644 test-orchestrator/modules/base/bin/start_network_service create mode 100644 test-orchestrator/modules/base/bin/wait_for_interface create mode 100644 test-orchestrator/modules/base/conf/module_config.json create mode 100644 test-orchestrator/modules/base/python/requirements.txt create mode 100644 test-orchestrator/modules/base/python/src/grpc/start_server.py create mode 100644 test-orchestrator/modules/base/python/src/logger.py create mode 100644 test-orchestrator/modules/template/bin/start_network_service create mode 100644 test-orchestrator/modules/template/conf/module_config.json create mode 100644 test-orchestrator/modules/template/python/src/logger.py create mode 100644 test-orchestrator/modules/template/python/src/run.py create mode 100644 test-orchestrator/modules/template/python/src/test_module.py create mode 100644 test-orchestrator/modules/template/template.Dockerfile create mode 100644 test-orchestrator/python/requirements.txt create mode 100644 test-orchestrator/python/src/docker_control.py create mode 100644 test-orchestrator/python/src/logger.py create mode 100644 test-orchestrator/python/src/run.py create mode 100644 test-orchestrator/python/src/test_orchestrator.py diff --git a/.gitignore b/.gitignore index 93fe84e64..2b0acd7e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,133 @@ venv/ net_orc/ -.vscode/ \ No newline at end of file +.vscode/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cmd/install b/cmd/install index 351eb4129..61722e273 100755 --- a/cmd/install +++ b/cmd/install @@ -2,6 +2,7 @@ GIT_URL=https://github.com/auto-iot NET_ORC_DIR=net_orc +NET_ORC_VERSION="dev" python3 -m venv venv @@ -10,7 +11,7 @@ source venv/bin/activate pip3 install -r etc/requirements.txt rm -rf $NET_ORC_DIR -git clone $GIT_URL/network-orchestrator $NET_ORC_DIR +git clone -b $NET_ORC_VERSION $GIT_URL/network-orchestrator $NET_ORC_DIR chown -R $USER $NET_ORC_DIR pip3 install -r $NET_ORC_DIR/python/requirements.txt diff --git a/conf/system.json.example b/conf/system.json.example index 379545ad6..9f3fcd523 100644 --- a/conf/system.json.example +++ b/conf/system.json.example @@ -1,7 +1,12 @@ { - "network": { - "device_intf": "enx123456789123", - "internet_intf": "enx123456789124" - }, - "log_level": "INFO" + "network": { + "device_intf": "enx123456789123", + "internet_intf": "enx123456789124" + }, + "modules": [ + { + "template": true + } + ], + "log_level": "INFO" } \ No newline at end of file diff --git a/framework/testrun.py b/framework/testrun.py index 225bed853..636a60805 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -22,7 +22,12 @@ net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') sys.path.append(net_orc_dir) +#Add test_orc to Python path +test_orc_dir = os.path.join(parent_dir, 'test-orchestrator', 'python', 'src') +sys.path.append(test_orc_dir) + import network_orchestrator as net_orc # pylint: disable=wrong-import-position +import test_orchestrator as test_orc LOGGER = logger.get_logger('test_run') CONFIG_FILE = "conf/system.json" @@ -71,11 +76,15 @@ def _load_config(self): with open(CONFIG_FILE, 'r', encoding='UTF-8') as config_file_open: config_json = json.load(config_file_open) self._net_orc.import_config(config_json) + self._test_orc.import_config(config_json) def _start_network(self): # Create an instance of the network orchestrator self._net_orc = net_orc.NetworkOrchestrator() + # Create an instance of the test orchestrator + self._test_orc=test_orc.TestOrchestrator() + # Load config file and pass to other components self._load_config() @@ -83,6 +92,9 @@ def _start_network(self): self._net_orc.load_network_modules() self._net_orc.build_network_modules() + # Load and build any unbuilt test module containers + self._test_orc.build_modules() + # Create baseline network self._net_orc.create_net() @@ -91,8 +103,11 @@ def _start_network(self): LOGGER.info("Network is ready.") + self._test_orc.start_modules() + def _stop_network(self): LOGGER.info("Stopping Test Run") self._net_orc.stop_networking_services(kill=True) + self._test_orc.stop_modules(kill=True) self._net_orc.restore_net() sys.exit(0) diff --git a/test-orchestrator/modules/base/base.Dockerfile b/test-orchestrator/modules/base/base.Dockerfile new file mode 100644 index 000000000..7edbb1bbd --- /dev/null +++ b/test-orchestrator/modules/base/base.Dockerfile @@ -0,0 +1,23 @@ +# Image name: test-run/base +FROM ubuntu:jammy + +# Install common software +RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix + +#Setup the base python requirements +COPY modules/base/python /testrun/python + +# Install all python requirements for the module +RUN pip3 install -r /testrun/python/requirements.txt + +# Add the bin files +COPY modules/base/bin /testrun/bin + +# Remove incorrect line endings +RUN dos2unix /testrun/bin/* + +# Make sure all the bin files are executable +RUN chmod u+x /testrun/bin/* + +#Start the network module +ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/capture b/test-orchestrator/modules/base/bin/capture new file mode 100644 index 000000000..8a8430feb --- /dev/null +++ b/test-orchestrator/modules/base/bin/capture @@ -0,0 +1,30 @@ +#!/bin/bash -e + +# Fetch module name +MODULE_NAME=$1 + +# Define the local file location for the capture to be saved +PCAP_DIR="/runtime/network/" +PCAP_FILE=$MODULE_NAME.pcap + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Allow a user to define an interface by passing it into this script +DEFINED_IFACE=$2 + +# Select which interace to use +if [[ -z $DEFINED_IFACE ]] +then + INTERFACE=$DEFAULT_IFACE +else + INTERFACE=$DEFINED_IFACE +fi + +# Create the output directory and start the capture +mkdir -p $PCAP_DIR +chown $HOST_USER:$HOST_USER $PCAP_DIR +tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & + +#Small pause to let the capture to start +sleep 1 \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/setup_binaries b/test-orchestrator/modules/base/bin/setup_binaries new file mode 100644 index 000000000..3535ead3c --- /dev/null +++ b/test-orchestrator/modules/base/bin/setup_binaries @@ -0,0 +1,10 @@ +#!/bin/bash -e + +# Directory where all binaries will be loaded +BIN_DIR=$1 + +# Remove incorrect line endings +dos2unix $BIN_DIR/* + +# Make sure all the bin files are executable +chmod u+x $BIN_DIR/* \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/start_grpc b/test-orchestrator/modules/base/bin/start_grpc new file mode 100644 index 000000000..9792b4bd4 --- /dev/null +++ b/test-orchestrator/modules/base/bin/start_grpc @@ -0,0 +1,17 @@ +#!/bin/bash -e + +GRPC_DIR="/testrun/python/src/grpc" +GRPC_PROTO_DIR="proto" +GRPC_PROTO_FILE="grpc.proto" + +#Move into the grpc directory +pushd $GRPC_DIR >/dev/null 2>&1 + +#Build the grpc proto file every time before starting server +python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. + +popd >/dev/null 2>&1 + +#Start the grpc server +python3 -u $GRPC_DIR/start_server.py $@ + diff --git a/test-orchestrator/modules/base/bin/start_module b/test-orchestrator/modules/base/bin/start_module new file mode 100644 index 000000000..7fdcbc404 --- /dev/null +++ b/test-orchestrator/modules/base/bin/start_module @@ -0,0 +1,79 @@ +#!/bin/bash + +# Directory where all binaries will be loaded +BIN_DIR="/testrun/bin" + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Create a local user that matches the same as the host +# to be used for correct file ownership for various logs +# HOST_USER mapped in via docker container environemnt variables +useradd $HOST_USER + +# Enable IPv6 for all containers +sysctl net.ipv6.conf.all.disable_ipv6=0 +sysctl -p + +#Read in the config file +CONF_FILE="/testrun/conf/module_config.json" +CONF=`cat $CONF_FILE` + +if [[ -z $CONF ]] +then + echo "No config file present at $CONF_FILE. Exiting startup." + exit 1 +fi + +# Extract the necessary config parameters +MODULE_NAME=$(echo "$CONF" | jq -r '.config.meta.name') +DEFINED_IFACE=$(echo "$CONF" | jq -r '.config.network.interface') +GRPC=$(echo "$CONF" | jq -r '.config.grpc') + +# Validate the module name is present +if [[ -z "$MODULE_NAME" || "$MODULE_NAME" == "null" ]] +then + echo "No module name present in $CONF_FILE. Exiting startup." + exit 1 +fi + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No Interface Defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +echo "Starting module $MODULE_NAME on local interface $INTF..." + +$BIN_DIR/setup_binaries $BIN_DIR + +# Wait for interface to become ready +$BIN_DIR/wait_for_interface $INTF + +# Small pause to let the interface stabalize before starting the capture +#sleep 1 + +# Start network capture +$BIN_DIR/capture $MODULE_NAME $INTF + +# Start the grpc server +if [[ ! -z $GRPC && ! $GRPC == "null" ]] +then + GRPC_PORT=$(echo "$GRPC" | jq -r '.port') + if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] + then + echo "gRPC port resolved from config: $GRPC_PORT" + $BIN_DIR/start_grpc "-p $GRPC_PORT" & + else + $BIN_DIR/start_grpc & + fi +fi + +#Small pause to let all core services stabalize +sleep 3 + +#Start the networking service +$BIN_DIR/start_network_service $MODULE_NAME $INTF \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/start_network_service b/test-orchestrator/modules/base/bin/start_network_service new file mode 100644 index 000000000..9f2336afa --- /dev/null +++ b/test-orchestrator/modules/base/bin/start_network_service @@ -0,0 +1,13 @@ +#!/bin/bash + +# Place holder function for testing and validation +# Each network module should include a start_networkig_service +# file that overwrites this one to boot all of the its specific +# requirements to run. + +echo "Starting Network Service..." +echo " This is not a real network service, just a test" +echo "Network Service Started" + +# Do Nothing, just keep the module alive +while true;do sleep 1; done \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/wait_for_interface b/test-orchestrator/modules/base/bin/wait_for_interface new file mode 100644 index 000000000..9135edf14 --- /dev/null +++ b/test-orchestrator/modules/base/bin/wait_for_interface @@ -0,0 +1,22 @@ +#!/bin/bash + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Allow a user to define an interface by passing it into this script +DEFINED_IFACE=$1 + +# Select which interace to use +if [[ -z $DEFINED_IFACE ]] +then + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + + +# Wait for local interface to be ready +while ! ip link show $INTF; do + echo $INTF is not yet ready. Waiting 3 seconds + sleep 3 +done \ No newline at end of file diff --git a/test-orchestrator/modules/base/conf/module_config.json b/test-orchestrator/modules/base/conf/module_config.json new file mode 100644 index 000000000..1f3a47ba2 --- /dev/null +++ b/test-orchestrator/modules/base/conf/module_config.json @@ -0,0 +1,12 @@ +{ + "config": { + "meta": { + "name": "base", + "display_name": "Base", + "description": "Base image" + }, + "docker": { + "enable_container": false + } + } +} \ No newline at end of file diff --git a/test-orchestrator/modules/base/python/requirements.txt b/test-orchestrator/modules/base/python/requirements.txt new file mode 100644 index 000000000..9c4e2b056 --- /dev/null +++ b/test-orchestrator/modules/base/python/requirements.txt @@ -0,0 +1,2 @@ +grpcio +grpcio-tools \ No newline at end of file diff --git a/test-orchestrator/modules/base/python/src/grpc/start_server.py b/test-orchestrator/modules/base/python/src/grpc/start_server.py new file mode 100644 index 000000000..9ed31ffcf --- /dev/null +++ b/test-orchestrator/modules/base/python/src/grpc/start_server.py @@ -0,0 +1,34 @@ +from concurrent import futures +import grpc +import proto.grpc_pb2_grpc as pb2_grpc +import proto.grpc_pb2 as pb2 +from network_service import NetworkService +import logging +import sys +import argparse + +DEFAULT_PORT = '5001' + +def serve(PORT): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + pb2_grpc.add_NetworkModuleServicer_to_server(NetworkService(), server) + server.add_insecure_port('[::]:' + PORT) + server.start() + server.wait_for_termination() + +def run(argv): + parser = argparse.ArgumentParser(description="GRPC Server for Network Module", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-p", "--port", default=DEFAULT_PORT, + help="Define the default port to run the server on.") + + args = parser.parse_args() + + PORT = args.port + + print("gRPC server starting on port " + PORT) + serve(PORT) + + +if __name__ == "__main__": + run(sys.argv) \ No newline at end of file diff --git a/test-orchestrator/modules/base/python/src/logger.py b/test-orchestrator/modules/base/python/src/logger.py new file mode 100644 index 000000000..4924512c6 --- /dev/null +++ b/test-orchestrator/modules/base/python/src/logger.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import json +import logging +import os + +LOGGERS = {} +_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" +_DATE_FORMAT = '%b %02d %H:%M:%S' +_DEFAULT_LEVEL = logging.INFO +_CONF_DIR = "conf" +_CONF_FILE_NAME = "system.json" +_LOG_DIR = "/runtime/network/" + +# Set log level +try: + system_conf_json = json.load( + open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) + log_level_str = system_conf_json['log_level'] + log_level = logging.getLevelName(log_level_str) +except: + # TODO: Print out warning that log level is incorrect or missing + log_level = _DEFAULT_LEVEL + +log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) + + +def add_file_handler(log, logFile): + handler = logging.FileHandler(_LOG_DIR+logFile+".log") + handler.setFormatter(log_format) + log.addHandler(handler) + + +def add_stream_handler(log): + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) + + +def get_logger(name, logFile=None): + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(log_level) + add_stream_handler(LOGGERS[name]) + if logFile is not None: + add_file_handler(LOGGERS[name], logFile) + return LOGGERS[name] diff --git a/test-orchestrator/modules/template/bin/start_network_service b/test-orchestrator/modules/template/bin/start_network_service new file mode 100644 index 000000000..3e1053700 --- /dev/null +++ b/test-orchestrator/modules/template/bin/start_network_service @@ -0,0 +1,40 @@ +#!/bin/bash + +# An example startup script that does the bare minimum to start +# a test module via a pyhon script. Each test module should include a +# start_network_service file that overwrites this one to boot all of its +#specific requirements to run. + +echo "Starting Network Service..." +echo " This is not a real network service, just a test" + +#Define where all the python source files are located +PYTHON_SRC_DIR=/testrun/python/src + +# Fetch module name +MODULE_NAME=$1 + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Allow a user to define an interface by passing it into this script +DEFINED_IFACE=$2 + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No Interface Defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +#Create and set permissions on the log files +LOG_FILE=/runtime/network/$MODULE_NAME.log +touch $LOG_FILE +chown $HOST_USER:$HOST_USER $LOG_FILE + +# Run the python scrip that will execute the tests for this module +# -u flag allows python print statements +# to be logged by docker by running unbuffered +python3 -u $PYTHON_SRC_DIR/run.py "-m $MODULE_NAME" \ No newline at end of file diff --git a/test-orchestrator/modules/template/conf/module_config.json b/test-orchestrator/modules/template/conf/module_config.json new file mode 100644 index 000000000..845f34521 --- /dev/null +++ b/test-orchestrator/modules/template/conf/module_config.json @@ -0,0 +1,26 @@ +{ + "config": { + "meta": { + "name": "template", + "display_name": "Template", + "description": "Template for building network service modules" + }, + "network": { + "interface": "eth0", + "enable_wan": false, + "ip_index": 9 + }, + "grpc": { + "port": 50001 + }, + "docker": { + "enable_container": true, + "mounts": [ + { + "source": "runtime/network", + "target": "/runtime/network" + } + ] + } + } +} \ No newline at end of file diff --git a/test-orchestrator/modules/template/python/src/logger.py b/test-orchestrator/modules/template/python/src/logger.py new file mode 100644 index 000000000..a9574e2ce --- /dev/null +++ b/test-orchestrator/modules/template/python/src/logger.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import json +import logging +import os +import sys + +LOGGERS = {} +_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" +_DATE_FORMAT = '%b %02d %H:%M:%S' +_DEFAULT_LEVEL = logging.INFO +_CONF_DIR = "conf" +_CONF_FILE_NAME = "system.json" +_LOG_DIR = "/runtime/network/" + +# Set log level +try: + system_conf_json = json.load( + open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) + log_level_str = system_conf_json['log_level'] + log_level = logging.getLevelName(log_level_str) +except: + # TODO: Print out warning that log level is incorrect or missing + log_level = _DEFAULT_LEVEL + +log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) + + +def add_file_handler(log, logFile): + handler = logging.FileHandler(_LOG_DIR+logFile+".log") + handler.setFormatter(log_format) + log.addHandler(handler) + + +def add_stream_handler(log): + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) + + +def get_logger(name, logFile=None): + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(log_level) + add_stream_handler(LOGGERS[name]) + if logFile is not None: + add_file_handler(LOGGERS[name], logFile) + return LOGGERS[name] diff --git a/test-orchestrator/modules/template/python/src/run.py b/test-orchestrator/modules/template/python/src/run.py new file mode 100644 index 000000000..dcf2a9861 --- /dev/null +++ b/test-orchestrator/modules/template/python/src/run.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import argparse +import signal +import sys +import time +import logger + +from test_module import TestModule + +LOGGER = logger.get_logger('test_module') +RUNTIME = 300 + +class TestModuleRunner: + + def __init__(self,module): + + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) + + LOGGER.info("Starting Test Module Template") + + self._test_module = TestModule(module) + self._test_module.run_tests() + self._test_module.generate_results() + + def _handler(self, signum, *other): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping test module...") + LOGGER.info("Test module stopped") + sys.exit(1) + +def run(argv): + parser = argparse.ArgumentParser(description="Test Module Template", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument( + "-m", "--module", help="Define the module name to be used to create the log file") + + args = parser.parse_args() + + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + TestModuleRunner(args.module.strip()) + +if __name__ == "__main__": + run(sys.argv) diff --git a/test-orchestrator/modules/template/python/src/test_module.py b/test-orchestrator/modules/template/python/src/test_module.py new file mode 100644 index 000000000..f5bbf9488 --- /dev/null +++ b/test-orchestrator/modules/template/python/src/test_module.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import signal +import time +import sys +import argparse +import logger + +LOGGER = None +LOG_NAME = "test_module_template" + + +class TestModule: + + def __init__(self, module): + + self.module_test1 = None + self.module_test2 = None + self.module_test3 = None + self.add_logger(module) + + signal.signal(signal.SIGINT, self.handler) + signal.signal(signal.SIGTERM, self.handler) + signal.signal(signal.SIGABRT, self.handler) + signal.signal(signal.SIGQUIT, self.handler) + + def add_logger(self, module): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, module) + + # Make up some fake test results + def run_tests(self): + LOGGER.info("Running test 1...") + self.module_test1 = True + LOGGER.info("Test 1 complete.") + + LOGGER.info("Running test 2...") + self.module_test2 = False + LOGGER.info("Test 2 complete.") + + def generate_results(self): + self.print_test_result("Test 1", self.module_test1) + self.print_test_result("Test 2", self.module_test2) + self.print_test_result("Test 3", self.module_test3) + + def print_test_result(self, test_name, result): + if result is not None: + LOGGER.info( + test_name + ": Pass" if result else test_name + ": Fail") + else: + LOGGER.info(test_name + " Skipped") + + def handler(self, signum, frame): + if (signum == 2 or signal == signal.SIGTERM): + exit(1) diff --git a/test-orchestrator/modules/template/template.Dockerfile b/test-orchestrator/modules/template/template.Dockerfile new file mode 100644 index 000000000..83aa5fcec --- /dev/null +++ b/test-orchestrator/modules/template/template.Dockerfile @@ -0,0 +1,11 @@ +# Image name: test-run/dhcp-primary +FROM test-run/base:latest + +# Copy over all configuration files +COPY modules/template/conf /testrun/conf + +# Load device binary files +COPY modules/template/bin /testrun/bin + +# Copy over all python files +COPY modules/template/python /testrun/python \ No newline at end of file diff --git a/test-orchestrator/python/requirements.txt b/test-orchestrator/python/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/test-orchestrator/python/src/docker_control.py b/test-orchestrator/python/src/docker_control.py new file mode 100644 index 000000000..92c4c8ff1 --- /dev/null +++ b/test-orchestrator/python/src/docker_control.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +import logger +import docker +import os +import json +from docker.types import Mount + +LOGGER = logger.get_logger('docker_cntl') +MODULES_DIR = "modules" +MODULE_CONFIG = "conf/module_config.json" + + +class DockerControl: + + def __init__(self): + self._modules = [] + self._path = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__)))) + + def _build_modules(self): + LOGGER.info("Building docker images...") + for module in self._modules: + self._build_module(module) + + def _build_module(self, module): + LOGGER.debug("Building docker image for module " + module.dir_name) + client = docker.from_env() + client.images.build( + dockerfile=os.path.join(module.dir, module.build_file), + path=self._path, + forcerm=True,# Cleans up intermediate containers during build + tag=module.image_name + ) + + def _get_module_container(self, module): + LOGGER.debug("Resolving test module container: " + + module.container_name) + container = None + try: + client = docker.from_env() + container = client.containers.get(module.container_name) + except docker.errors.NotFound: + LOGGER.debug("Container " + + module.container_name + " not found") + except Exception as e: + LOGGER.error("Failed to resolve container") + LOGGER.error(e) + return container + + def _start_modules(self): + LOGGER.info("Starting test modules") + for module in self._modules: + # Test modules may just be Docker images, so we do not want to start them as containers + if not module.enable_container: + continue + + self._start_module(module) + + LOGGER.info("All network services are running") + + def _start_module(self, module): + + LOGGER.debug("Starting test module " + module.display_name) + try: + client = docker.from_env() + module.container = client.containers.run( + module.image_name, + auto_remove=True, + cap_add=["NET_ADMIN"], + name=module.container_name, + hostname=module.container_name, + privileged=True, + detach=True, + mounts=module.mounts, + environment={"HOST_USER": os.getlogin()} + ) + except Exception as client_error: + LOGGER.error("Container run error") + LOGGER.error(client_error) + + def _stop_modules(self, kill=False): + LOGGER.info("Stopping test modules") + for module in self._modules: + # Test modules may just be Docker images, so we do not want to stop them + if not module.enable_container: + continue + self._stop_module(module, kill) + + def _stop_module(self, module, kill=False): + LOGGER.debug("Stopping test module " + module.container_name) + try: + container = self._get_module_container(module) + if container is not None: + if kill: + LOGGER.debug("Killing container:" + + module.container_name) + container.kill() + else: + LOGGER.debug("Stopping container:" + + module.container_name) + container.stop() + LOGGER.debug("Container stopped:" + module.container_name) + except Exception as error: + LOGGER.error("Container stop error") + LOGGER.error(error) + + + def load_modules(self): + modules_dir = os.path.join(self._path, MODULES_DIR) + LOGGER.debug("Loading modules from /" + modules_dir) + + loaded_modules = "Loaded the following test modules: " + for module_dir in os.listdir(modules_dir): + + LOGGER.debug("Loading Module from: " + module_dir) + # Load basic module information + module = Module() + module_json = json.load(open(os.path.join( + self._path, modules_dir, module_dir, MODULE_CONFIG), encoding='UTF-8')) + LOGGER.debug(module_json) + module.name = module_json['config']['meta']['name'] + module.display_name = module_json['config']['meta']['display_name'] + module.description = module_json['config']['meta']['description'] + module.dir = os.path.join( + self._path, modules_dir, module_dir) + module.dir_name = module_dir + module.build_file = module_dir + ".Dockerfile" + module.container_name = "tr-ct-" + module.dir_name + "-test" + module.image_name = "test-run/" + module.dir_name + "-test" + + # Attach folder mounts to network module + if "docker" in module_json['config']: + if "mounts" in module_json['config']['docker']: + for mount_point in module_json['config']['docker']['mounts']: + module.mounts.append(Mount( + target=mount_point['target'], + source=os.path.join( + self._path, mount_point['source']), + type='bind' + )) + + # Determine if this is a container or just an image/template + if "enable_container" in module_json['config']['docker']: + module.enable_container = module_json['config']['docker']['enable_container'] + + self._modules.append(module) + + loaded_modules += module.dir_name + " " + + LOGGER.info(loaded_modules) + + +class Module: + + def __init__(self): + self.name = None + self.display_name = None + self.description = None + + self.container = None + self.container_name = None + self.image_name = None + + # Absolute path + self.dir = None + self.dir_name = None + self.build_file = None + self.mounts = [] + + self.enable_container = True diff --git a/test-orchestrator/python/src/logger.py b/test-orchestrator/python/src/logger.py new file mode 100644 index 000000000..a9574e2ce --- /dev/null +++ b/test-orchestrator/python/src/logger.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import json +import logging +import os +import sys + +LOGGERS = {} +_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" +_DATE_FORMAT = '%b %02d %H:%M:%S' +_DEFAULT_LEVEL = logging.INFO +_CONF_DIR = "conf" +_CONF_FILE_NAME = "system.json" +_LOG_DIR = "/runtime/network/" + +# Set log level +try: + system_conf_json = json.load( + open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) + log_level_str = system_conf_json['log_level'] + log_level = logging.getLevelName(log_level_str) +except: + # TODO: Print out warning that log level is incorrect or missing + log_level = _DEFAULT_LEVEL + +log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) + + +def add_file_handler(log, logFile): + handler = logging.FileHandler(_LOG_DIR+logFile+".log") + handler.setFormatter(log_format) + log.addHandler(handler) + + +def add_stream_handler(log): + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) + + +def get_logger(name, logFile=None): + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(log_level) + add_stream_handler(LOGGERS[name]) + if logFile is not None: + add_file_handler(LOGGERS[name], logFile) + return LOGGERS[name] diff --git a/test-orchestrator/python/src/run.py b/test-orchestrator/python/src/run.py new file mode 100644 index 000000000..e2d371336 --- /dev/null +++ b/test-orchestrator/python/src/run.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import argparse +import signal +import sys +import time +import logger + +from test_orchestrator import TestOrchestrator + +LOGGER = logger.get_logger('net_orc') +RUNTIME = 300 + + +class TestRunner: + + def __init__(self, args): + + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) + + LOGGER.info("Starting Test Orchestrator") + + # Get all components ready + self._test_orc = TestOrchestrator() + self._test_orc.load_config() + # self._net_orc.load_network_modules() + + # # Restore the network first if required + # self._net_orc.stop_networking_services() + # self._net_orc.restore_net() + + # self._net_orc.build_network_modules() + # self._net_orc.create_net() + # self._net_orc.start_network_services() + + # Get network ready (via Network orchestrator) + LOGGER.info("Test orchestrator is ready.") + + # TODO: This time should be configurable (How long to hold before exiting, this could be infinite too) + time.sleep(RUNTIME) + + # self._validator._stop_validator() + # # Gracefully shutdown network + # self._net_orc.stop_networking_services(kill=False) + # self._net_orc.restore_net() + + def _handler(self, signum, *other): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping tests...") + # Kill all container services quickly + # If we're here, we want everything to stop immediately + # and don't care about a gracefully shutdown. + self._validator._stop_validator(True) + self._net_orc.stop_networking_services(True) + self._net_orc.restore_net() + LOGGER.info("Tests stopped") + sys.exit(1) + + +def run(argv): + parser = argparse.ArgumentParser(description="Test Run Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("--validate", action="store_true", + help="Run the validation of the network after network boot") + + args = parser.parse_args() + + TestRunner(args) + + +if __name__ == "__main__": + run(sys.argv) diff --git a/test-orchestrator/python/src/test_orchestrator.py b/test-orchestrator/python/src/test_orchestrator.py new file mode 100644 index 000000000..830e8cc9c --- /dev/null +++ b/test-orchestrator/python/src/test_orchestrator.py @@ -0,0 +1,40 @@ +from docker_control import DockerControl +import os +import logger +import json + +LOGGER = logger.get_logger('test_orc') +RUNTIME_DIR = "runtime/network" +TEST_MODULES_DIR = "tests/modules" +CONFIG_FILE = "conf/system.json" + + +class TestOrchestrator: + + def __init__(self): + self._test_modules = [] + self._module_config = None + + self._path = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__)))) + + # os.rmdir(os.path.join(self._path, RUNTIME_DIR)) + os.makedirs(os.path.join(self._path, RUNTIME_DIR), exist_ok=True) + + self._docker_cntrl = DockerControl() + + def start_modules(self): + self._docker_cntrl._start_modules() + + def stop_modules(self,kill=False): + self._docker_cntrl._stop_modules(kill=kill) + + def build_modules(self): + # Load and build any unbuilt network containers + self._docker_cntrl.load_modules() + self._docker_cntrl._build_modules() + + def import_config(self, json_config): + modules = json_config['modules'] + for module in modules: + print(str(module)) From 28075410464284ce56b1d246207f9ff7b986a6a6 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 08:27:44 -0600 Subject: [PATCH 02/20] Ignore runtime folder --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2b0acd7e1..4016b6901 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# Runtime folder +runtime/ venv/ net_orc/ .vscode/ From 50b75fcf71706868cd96453d3f02fece1ac458a2 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 09:41:25 -0600 Subject: [PATCH 03/20] Update runtime directory for test modules --- test-orchestrator/modules/base/bin/capture | 3 ++- .../modules/template/conf/module_config.json | 4 ++-- test-orchestrator/modules/template/python/src/logger.py | 2 +- test-orchestrator/modules/template/template.Dockerfile | 2 +- test-orchestrator/python/src/docker_control.py | 9 ++++++++- test-orchestrator/python/src/logger.py | 2 +- test-orchestrator/python/src/test_orchestrator.py | 7 +++++-- 7 files changed, 20 insertions(+), 9 deletions(-) diff --git a/test-orchestrator/modules/base/bin/capture b/test-orchestrator/modules/base/bin/capture index 8a8430feb..45c627040 100644 --- a/test-orchestrator/modules/base/bin/capture +++ b/test-orchestrator/modules/base/bin/capture @@ -4,7 +4,7 @@ MODULE_NAME=$1 # Define the local file location for the capture to be saved -PCAP_DIR="/runtime/network/" +PCAP_DIR="/runtime/testing/" PCAP_FILE=$MODULE_NAME.pcap # Default interface should be veth0 for all containers @@ -24,6 +24,7 @@ fi # Create the output directory and start the capture mkdir -p $PCAP_DIR chown $HOST_USER:$HOST_USER $PCAP_DIR +echo "PCAP Dir: $PCAP_DIR/$PCAP_FILE" tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & #Small pause to let the capture to start diff --git a/test-orchestrator/modules/template/conf/module_config.json b/test-orchestrator/modules/template/conf/module_config.json index 845f34521..6b276ef6d 100644 --- a/test-orchestrator/modules/template/conf/module_config.json +++ b/test-orchestrator/modules/template/conf/module_config.json @@ -17,8 +17,8 @@ "enable_container": true, "mounts": [ { - "source": "runtime/network", - "target": "/runtime/network" + "source": "runtime/testing", + "target": "/runtime/testing" } ] } diff --git a/test-orchestrator/modules/template/python/src/logger.py b/test-orchestrator/modules/template/python/src/logger.py index a9574e2ce..7294b77fd 100644 --- a/test-orchestrator/modules/template/python/src/logger.py +++ b/test-orchestrator/modules/template/python/src/logger.py @@ -11,7 +11,7 @@ _DEFAULT_LEVEL = logging.INFO _CONF_DIR = "conf" _CONF_FILE_NAME = "system.json" -_LOG_DIR = "/runtime/network/" +_LOG_DIR = "/runtime/testing/" # Set log level try: diff --git a/test-orchestrator/modules/template/template.Dockerfile b/test-orchestrator/modules/template/template.Dockerfile index 83aa5fcec..bdbb79c41 100644 --- a/test-orchestrator/modules/template/template.Dockerfile +++ b/test-orchestrator/modules/template/template.Dockerfile @@ -1,5 +1,5 @@ # Image name: test-run/dhcp-primary -FROM test-run/base:latest +FROM test-run/base-test:latest # Copy over all configuration files COPY modules/template/conf /testrun/conf diff --git a/test-orchestrator/python/src/docker_control.py b/test-orchestrator/python/src/docker_control.py index 92c4c8ff1..850d1d59a 100644 --- a/test-orchestrator/python/src/docker_control.py +++ b/test-orchestrator/python/src/docker_control.py @@ -15,8 +15,15 @@ class DockerControl: def __init__(self): self._modules = [] + + #Resolve the path to the test-orchestrator folder self._path = os.path.dirname(os.path.dirname( os.path.dirname(os.path.realpath(__file__)))) + + #Resolve the path to the test-run folder + self._root_path = os.path.abspath(os.path.join(self._path,os.pardir)) + + LOGGER.info("Orchestrator path: " + self._root_path) def _build_modules(self): LOGGER.info("Building docker images...") @@ -136,7 +143,7 @@ def load_modules(self): module.mounts.append(Mount( target=mount_point['target'], source=os.path.join( - self._path, mount_point['source']), + self._root_path, mount_point['source']), type='bind' )) diff --git a/test-orchestrator/python/src/logger.py b/test-orchestrator/python/src/logger.py index a9574e2ce..7294b77fd 100644 --- a/test-orchestrator/python/src/logger.py +++ b/test-orchestrator/python/src/logger.py @@ -11,7 +11,7 @@ _DEFAULT_LEVEL = logging.INFO _CONF_DIR = "conf" _CONF_FILE_NAME = "system.json" -_LOG_DIR = "/runtime/network/" +_LOG_DIR = "/runtime/testing/" # Set log level try: diff --git a/test-orchestrator/python/src/test_orchestrator.py b/test-orchestrator/python/src/test_orchestrator.py index 830e8cc9c..1b0681bc1 100644 --- a/test-orchestrator/python/src/test_orchestrator.py +++ b/test-orchestrator/python/src/test_orchestrator.py @@ -4,7 +4,7 @@ import json LOGGER = logger.get_logger('test_orc') -RUNTIME_DIR = "runtime/network" +RUNTIME_DIR = "runtime/testing" TEST_MODULES_DIR = "tests/modules" CONFIG_FILE = "conf/system.json" @@ -18,8 +18,11 @@ def __init__(self): self._path = os.path.dirname(os.path.dirname( os.path.dirname(os.path.realpath(__file__)))) + #Resolve the path to the test-run folder + self._root_path = os.path.abspath(os.path.join(self._path,os.pardir)) + # os.rmdir(os.path.join(self._path, RUNTIME_DIR)) - os.makedirs(os.path.join(self._path, RUNTIME_DIR), exist_ok=True) + os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) self._docker_cntrl = DockerControl() From 88d553f9bb1faf9287245a31bbcfb1a06ba57407 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 15:40:41 -0600 Subject: [PATCH 04/20] Fix logging Add initial framework for running tests --- conf/system.json.example | 3 +- framework/logger.py | 40 ++++++++++++--- framework/testrun.py | 2 +- .../python/src/docker_control.py | 22 +++++++- test-orchestrator/python/src/logger.py | 48 ------------------ .../python/src/test_orchestrator.py | 50 ++++++++++++++++--- 6 files changed, 98 insertions(+), 67 deletions(-) delete mode 100644 test-orchestrator/python/src/logger.py diff --git a/conf/system.json.example b/conf/system.json.example index 9f3fcd523..f7212ceb3 100644 --- a/conf/system.json.example +++ b/conf/system.json.example @@ -5,7 +5,8 @@ }, "modules": [ { - "template": true + "name": "template", + "enabled": true } ], "log_level": "INFO" diff --git a/framework/logger.py b/framework/logger.py index 25970bd21..db4747af5 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -1,23 +1,47 @@ -"""Manages all things logging.""" +#!/usr/bin/env python3 + import json import logging import os +import sys LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" _DATE_FORMAT = '%b %02d %H:%M:%S' -_CONF_DIR="conf" -_CONF_FILE_NAME="system.json" +_DEFAULT_LEVEL = logging.INFO +_CONF_DIR = "conf" +_CONF_FILE_NAME = "system.json" +_LOG_DIR = "runtime/testing/" -with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as config_file: - system_conf_json = json.load(config_file) +# Set log level +try: + system_conf_json = json.load( + open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) log_level_str = system_conf_json['log_level'] log_level = logging.getLevelName(log_level_str) +except: + # TODO: Print out warning that log level is incorrect or missing + log_level = _DEFAULT_LEVEL + +log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) + + +def add_file_handler(log, logFile): + handler = logging.FileHandler(_LOG_DIR+logFile+".log") + handler.setFormatter(log_format) + log.addHandler(handler) + -logging.basicConfig(format=_LOG_FORMAT, datefmt=_DATE_FORMAT, level=log_level) +def add_stream_handler(log): + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) -def get_logger(name): - """Returns the logger belonging to the class calling the method.""" +def get_logger(name, logFile=None): if name not in LOGGERS: LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(log_level) + add_stream_handler(LOGGERS[name]) + if logFile is not None: + add_file_handler(LOGGERS[name], logFile) return LOGGERS[name] diff --git a/framework/testrun.py b/framework/testrun.py index 636a60805..313ba94b3 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -103,7 +103,7 @@ def _start_network(self): LOGGER.info("Network is ready.") - self._test_orc.start_modules() + self._test_orc.run_test_modules() def _stop_network(self): LOGGER.info("Stopping Test Run") diff --git a/test-orchestrator/python/src/docker_control.py b/test-orchestrator/python/src/docker_control.py index 850d1d59a..2ef405cc9 100644 --- a/test-orchestrator/python/src/docker_control.py +++ b/test-orchestrator/python/src/docker_control.py @@ -6,14 +6,15 @@ import json from docker.types import Mount -LOGGER = logger.get_logger('docker_cntl') +LOG_NAME = "docker_cntl" +LOGGER = None MODULES_DIR = "modules" MODULE_CONFIG = "conf/module_config.json" class DockerControl: - def __init__(self): + def __init__(self,module): self._modules = [] #Resolve the path to the test-orchestrator folder @@ -23,8 +24,13 @@ def __init__(self): #Resolve the path to the test-run folder self._root_path = os.path.abspath(os.path.join(self._path,os.pardir)) + self.add_logger(module) LOGGER.info("Orchestrator path: " + self._root_path) + def add_logger(self, module): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, module) + def _build_modules(self): LOGGER.info("Building docker images...") for module in self._modules: @@ -40,6 +46,18 @@ def _build_module(self, module): tag=module.image_name ) + def _get_module(self,name): + for module in self._modules: + if name == module.name: + return module + return None + + def _get_module_status(self,module): + container = self._get_module_container(module) + if container is not None: + return container.status + return None + def _get_module_container(self, module): LOGGER.debug("Resolving test module container: " + module.container_name) diff --git a/test-orchestrator/python/src/logger.py b/test-orchestrator/python/src/logger.py deleted file mode 100644 index 7294b77fd..000000000 --- a/test-orchestrator/python/src/logger.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -import json -import logging -import os -import sys - -LOGGERS = {} -_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" -_DATE_FORMAT = '%b %02d %H:%M:%S' -_DEFAULT_LEVEL = logging.INFO -_CONF_DIR = "conf" -_CONF_FILE_NAME = "system.json" -_LOG_DIR = "/runtime/testing/" - -# Set log level -try: - system_conf_json = json.load( - open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) - log_level_str = system_conf_json['log_level'] - log_level = logging.getLevelName(log_level_str) -except: - # TODO: Print out warning that log level is incorrect or missing - log_level = _DEFAULT_LEVEL - -log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) - - -def add_file_handler(log, logFile): - handler = logging.FileHandler(_LOG_DIR+logFile+".log") - handler.setFormatter(log_format) - log.addHandler(handler) - - -def add_stream_handler(log): - handler = logging.StreamHandler() - handler.setFormatter(log_format) - log.addHandler(handler) - - -def get_logger(name, logFile=None): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(log_level) - add_stream_handler(LOGGERS[name]) - if logFile is not None: - add_file_handler(LOGGERS[name], logFile) - return LOGGERS[name] diff --git a/test-orchestrator/python/src/test_orchestrator.py b/test-orchestrator/python/src/test_orchestrator.py index 1b0681bc1..a018e77e2 100644 --- a/test-orchestrator/python/src/test_orchestrator.py +++ b/test-orchestrator/python/src/test_orchestrator.py @@ -2,11 +2,15 @@ import os import logger import json +import time +import shutil -LOGGER = logger.get_logger('test_orc') +LOG_NAME = "test_orc" +LOGGER = logger.get_logger('LOG_NAME') RUNTIME_DIR = "runtime/testing" TEST_MODULES_DIR = "tests/modules" CONFIG_FILE = "conf/system.json" +MODULE_NAME = "test_orchestrator" class TestOrchestrator: @@ -18,18 +22,50 @@ def __init__(self): self._path = os.path.dirname(os.path.dirname( os.path.dirname(os.path.realpath(__file__)))) - #Resolve the path to the test-run folder - self._root_path = os.path.abspath(os.path.join(self._path,os.pardir)) + # Resolve the path to the test-run folder + self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) - # os.rmdir(os.path.join(self._path, RUNTIME_DIR)) + shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR)) os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) - self._docker_cntrl = DockerControl() + self.add_logger() + + self._docker_cntrl = DockerControl(MODULE_NAME) + + def add_logger(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, MODULE_NAME) + + def run_test_modules(self): + LOGGER.info("Running test modules") + for module in self._test_modules: + if module["enabled"]: + self.run_test_module(module) + + def run_test_module(self, module_config): + # Start the test container + LOGGER.info("Resolving container for test module: " + + module_config["name"]) + module = self._docker_cntrl._get_module(module_config["name"]) + if module is not None: + LOGGER.info("Test module container resolved: " + + module.container_name) + self._docker_cntrl._start_module(module) + + # Wait for the container to exit + status = self._docker_cntrl._get_module_status(module) + LOGGER.info("Test module " + module.name + " status: " + status) + if status == "running": + LOGGER.info("Waiting for test module " + module.name + " to complete") + while status == "running": + time.sleep(1) + status = self._docker_cntrl._get_module_status(module) + LOGGER.info("Test module " + module.name + " done") def start_modules(self): self._docker_cntrl._start_modules() - def stop_modules(self,kill=False): + def stop_modules(self, kill=False): self._docker_cntrl._stop_modules(kill=kill) def build_modules(self): @@ -40,4 +76,4 @@ def build_modules(self): def import_config(self, json_config): modules = json_config['modules'] for module in modules: - print(str(module)) + self._test_modules.append(module) From 0596204981c3d0e2c586177c811da7a2e9e86862 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 16:06:35 -0600 Subject: [PATCH 05/20] logging and misc cleanup --- framework/testrun.py | 226 +++++++++--------- .../python/src/docker_control.py | 1 - test-orchestrator/python/src/run.py | 77 ------ .../python/src/test_orchestrator.py | 9 +- 4 files changed, 117 insertions(+), 196 deletions(-) delete mode 100644 test-orchestrator/python/src/run.py diff --git a/framework/testrun.py b/framework/testrun.py index 313ba94b3..639d4adc2 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -1,113 +1,113 @@ -"""The overall control of the Test Run application. - -This file provides the integration between all of the -Test Run components, such as net_orc, test_orc and test_ui. - -Run using the provided command scripts in the cmd folder. -E.g sudo cmd/start -""" - -import os -import sys -import json -import signal -import time -import logger - -# Locate parent directory -current_dir = os.path.dirname(os.path.realpath(__file__)) -parent_dir = os.path.dirname(current_dir) - -# Add net_orc to Python path -net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') -sys.path.append(net_orc_dir) - -#Add test_orc to Python path -test_orc_dir = os.path.join(parent_dir, 'test-orchestrator', 'python', 'src') -sys.path.append(test_orc_dir) - -import network_orchestrator as net_orc # pylint: disable=wrong-import-position -import test_orchestrator as test_orc - -LOGGER = logger.get_logger('test_run') -CONFIG_FILE = "conf/system.json" -EXAMPLE_CONFIG_FILE = "conf/system.json.example" -RUNTIME = 300 - -class TestRun: # pylint: disable=too-few-public-methods - """Test Run controller. - - Creates an instance of the network orchestrator, test - orchestrator and user interface. - """ - - def __init__(self): - LOGGER.info("Starting Test Run") - - # Catch any exit signals - self._register_exits() - - self._start_network() - - # Keep application running - time.sleep(RUNTIME) - - self._stop_network() - - def _register_exits(self): - signal.signal(signal.SIGINT, self._exit_handler) - signal.signal(signal.SIGTERM, self._exit_handler) - signal.signal(signal.SIGABRT, self._exit_handler) - signal.signal(signal.SIGQUIT, self._exit_handler) - - def _exit_handler(self, signum, arg): # pylint: disable=unused-argument - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received.") - self._stop_network() - - def _load_config(self): - """Loads all settings from the config file into memory.""" - if not os.path.isfile(CONFIG_FILE): - LOGGER.error("Configuration file is not present at " + CONFIG_FILE) - LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) - sys.exit(1) - - with open(CONFIG_FILE, 'r', encoding='UTF-8') as config_file_open: - config_json = json.load(config_file_open) - self._net_orc.import_config(config_json) - self._test_orc.import_config(config_json) - - def _start_network(self): - # Create an instance of the network orchestrator - self._net_orc = net_orc.NetworkOrchestrator() - - # Create an instance of the test orchestrator - self._test_orc=test_orc.TestOrchestrator() - - # Load config file and pass to other components - self._load_config() - - # Load and build any unbuilt network containers - self._net_orc.load_network_modules() - self._net_orc.build_network_modules() - - # Load and build any unbuilt test module containers - self._test_orc.build_modules() - - # Create baseline network - self._net_orc.create_net() - - # Launch network service containers - self._net_orc.start_network_services() - - LOGGER.info("Network is ready.") - - self._test_orc.run_test_modules() - - def _stop_network(self): - LOGGER.info("Stopping Test Run") - self._net_orc.stop_networking_services(kill=True) - self._test_orc.stop_modules(kill=True) - self._net_orc.restore_net() - sys.exit(0) +"""The overall control of the Test Run application. + +This file provides the integration between all of the +Test Run components, such as net_orc, test_orc and test_ui. + +Run using the provided command scripts in the cmd folder. +E.g sudo cmd/start +""" + +import os +import sys +import json +import signal +import time +import logger + +# Locate parent directory +current_dir = os.path.dirname(os.path.realpath(__file__)) +parent_dir = os.path.dirname(current_dir) + +# Add net_orc to Python path +net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') +sys.path.append(net_orc_dir) + +#Add test_orc to Python path +test_orc_dir = os.path.join(parent_dir, 'test-orchestrator', 'python', 'src') +sys.path.append(test_orc_dir) + +import network_orchestrator as net_orc # pylint: disable=wrong-import-position +import test_orchestrator as test_orc + +LOGGER = logger.get_logger('test_run') +CONFIG_FILE = "conf/system.json" +EXAMPLE_CONFIG_FILE = "conf/system.json.example" +RUNTIME = 300 + +class TestRun: # pylint: disable=too-few-public-methods + """Test Run controller. + + Creates an instance of the network orchestrator, test + orchestrator and user interface. + """ + + def __init__(self): + LOGGER.info("Starting Test Run") + + # Catch any exit signals + self._register_exits() + + self._start_network() + + # Keep application running + time.sleep(RUNTIME) + + self._stop_network() + + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) + + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received.") + self._stop_network() + + def _load_config(self): + """Loads all settings from the config file into memory.""" + if not os.path.isfile(CONFIG_FILE): + LOGGER.error("Configuration file is not present at " + CONFIG_FILE) + LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) + sys.exit(1) + + with open(CONFIG_FILE, 'r', encoding='UTF-8') as config_file_open: + config_json = json.load(config_file_open) + self._net_orc.import_config(config_json) + self._test_orc.import_config(config_json) + + def _start_network(self): + # Create an instance of the network orchestrator + self._net_orc = net_orc.NetworkOrchestrator() + + # Create an instance of the test orchestrator + self._test_orc=test_orc.TestOrchestrator() + + # Load config file and pass to other components + self._load_config() + + # Load and build any unbuilt network containers + self._net_orc.load_network_modules() + self._net_orc.build_network_modules() + + # Load and build any unbuilt test module containers + self._test_orc.build_modules() + + # Create baseline network + self._net_orc.create_net() + + # Launch network service containers + self._net_orc.start_network_services() + + LOGGER.info("Network is ready.") + + self._test_orc.run_test_modules() + + def _stop_network(self): + LOGGER.info("Stopping Test Run") + self._net_orc.stop_networking_services(kill=True) + self._test_orc.stop_modules(kill=True) + self._net_orc.restore_net() + sys.exit(0) diff --git a/test-orchestrator/python/src/docker_control.py b/test-orchestrator/python/src/docker_control.py index 2ef405cc9..25061b064 100644 --- a/test-orchestrator/python/src/docker_control.py +++ b/test-orchestrator/python/src/docker_control.py @@ -25,7 +25,6 @@ def __init__(self,module): self._root_path = os.path.abspath(os.path.join(self._path,os.pardir)) self.add_logger(module) - LOGGER.info("Orchestrator path: " + self._root_path) def add_logger(self, module): global LOGGER diff --git a/test-orchestrator/python/src/run.py b/test-orchestrator/python/src/run.py deleted file mode 100644 index e2d371336..000000000 --- a/test-orchestrator/python/src/run.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import signal -import sys -import time -import logger - -from test_orchestrator import TestOrchestrator - -LOGGER = logger.get_logger('net_orc') -RUNTIME = 300 - - -class TestRunner: - - def __init__(self, args): - - signal.signal(signal.SIGINT, self._handler) - signal.signal(signal.SIGTERM, self._handler) - signal.signal(signal.SIGABRT, self._handler) - signal.signal(signal.SIGQUIT, self._handler) - - LOGGER.info("Starting Test Orchestrator") - - # Get all components ready - self._test_orc = TestOrchestrator() - self._test_orc.load_config() - # self._net_orc.load_network_modules() - - # # Restore the network first if required - # self._net_orc.stop_networking_services() - # self._net_orc.restore_net() - - # self._net_orc.build_network_modules() - # self._net_orc.create_net() - # self._net_orc.start_network_services() - - # Get network ready (via Network orchestrator) - LOGGER.info("Test orchestrator is ready.") - - # TODO: This time should be configurable (How long to hold before exiting, this could be infinite too) - time.sleep(RUNTIME) - - # self._validator._stop_validator() - # # Gracefully shutdown network - # self._net_orc.stop_networking_services(kill=False) - # self._net_orc.restore_net() - - def _handler(self, signum, *other): - LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received. Stopping tests...") - # Kill all container services quickly - # If we're here, we want everything to stop immediately - # and don't care about a gracefully shutdown. - self._validator._stop_validator(True) - self._net_orc.stop_networking_services(True) - self._net_orc.restore_net() - LOGGER.info("Tests stopped") - sys.exit(1) - - -def run(argv): - parser = argparse.ArgumentParser(description="Test Run Help", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("--validate", action="store_true", - help="Run the validation of the network after network boot") - - args = parser.parse_args() - - TestRunner(args) - - -if __name__ == "__main__": - run(sys.argv) diff --git a/test-orchestrator/python/src/test_orchestrator.py b/test-orchestrator/python/src/test_orchestrator.py index a018e77e2..307ca4898 100644 --- a/test-orchestrator/python/src/test_orchestrator.py +++ b/test-orchestrator/python/src/test_orchestrator.py @@ -6,13 +6,12 @@ import shutil LOG_NAME = "test_orc" -LOGGER = logger.get_logger('LOG_NAME') +LOGGER = None RUNTIME_DIR = "runtime/testing" TEST_MODULES_DIR = "tests/modules" CONFIG_FILE = "conf/system.json" MODULE_NAME = "test_orchestrator" - class TestOrchestrator: def __init__(self): @@ -28,13 +27,13 @@ def __init__(self): shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR)) os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) - self.add_logger() + self.add_logger(MODULE_NAME) self._docker_cntrl = DockerControl(MODULE_NAME) - def add_logger(self): + def add_logger(self,module): global LOGGER - LOGGER = logger.get_logger(LOG_NAME, MODULE_NAME) + LOGGER = logger.get_logger(LOG_NAME, module) def run_test_modules(self): LOGGER.info("Running test modules") From d3b46ccaeb0108ed6539124702116387934bd1d9 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 16:14:04 -0600 Subject: [PATCH 06/20] logging changes --- test-orchestrator/python/src/test_orchestrator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-orchestrator/python/src/test_orchestrator.py b/test-orchestrator/python/src/test_orchestrator.py index 307ca4898..c64c691ea 100644 --- a/test-orchestrator/python/src/test_orchestrator.py +++ b/test-orchestrator/python/src/test_orchestrator.py @@ -40,6 +40,7 @@ def run_test_modules(self): for module in self._test_modules: if module["enabled"]: self.run_test_module(module) + LOGGER.info("All tests complete") def run_test_module(self, module_config): # Start the test container @@ -53,13 +54,13 @@ def run_test_module(self, module_config): # Wait for the container to exit status = self._docker_cntrl._get_module_status(module) - LOGGER.info("Test module " + module.name + " status: " + status) + LOGGER.info("Test module " + module.display_name + " status: " + status) if status == "running": - LOGGER.info("Waiting for test module " + module.name + " to complete") + LOGGER.info("Waiting for test module " + module.display_name + " to complete") while status == "running": time.sleep(1) status = self._docker_cntrl._get_module_status(module) - LOGGER.info("Test module " + module.name + " done") + LOGGER.info("Test module " + module.display_name + " done") def start_modules(self): self._docker_cntrl._start_modules() From 2f671d98f3256f8438ede544d4a58c449bcea5e5 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 20 Apr 2023 16:15:55 -0600 Subject: [PATCH 07/20] Add a stop hook after all tests complete --- framework/testrun.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/framework/testrun.py b/framework/testrun.py index 639d4adc2..26a6eb5cd 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -103,8 +103,13 @@ def _start_network(self): LOGGER.info("Network is ready.") + # Begin testing self._test_orc.run_test_modules() + # Shutdown after testing completed + self._stop_network() + + def _stop_network(self): LOGGER.info("Stopping Test Run") self._net_orc.stop_networking_services(kill=True) From c7f31151ebdb32143c33d9292aad36744722ef1d Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Mon, 24 Apr 2023 13:48:35 +0100 Subject: [PATCH 08/20] Refactor test_orc code --- cmd/start | 6 + conf/system.json.example | 6 - framework/run.py | 21 +- framework/testrun.py | 51 ++--- .../modules/base/bin/start_network_service | 13 -- .../modules/base/bin/wait_for_interface | 22 -- .../modules/template/conf/module_config.json | 26 --- .../python/src/docker_control.py | 195 ---------------- .../python/src/test_orchestrator.py | 79 ------- .../modules/base/base.Dockerfile | 2 +- .../modules/base/bin/capture | 17 +- .../modules/base/bin/setup_binaries | 0 .../modules/base/bin/start_grpc | 6 +- .../modules/base/bin/start_module | 13 +- test_orc/modules/base/bin/wait_for_interface | 10 + .../modules/base/conf/module_config.json | 0 .../modules/base/python/requirements.txt | 0 .../base/python/src/grpc/start_server.py | 0 .../modules/base/python/src/logger.py | 0 .../modules/baseline/baseline.Dockerfile | 6 +- .../modules/baseline/bin/start_test_module | 13 +- .../modules/baseline/conf/module_config.json | 21 ++ .../modules/baseline}/python/src/logger.py | 4 +- .../modules/baseline}/python/src/run.py | 1 - .../baseline}/python/src/test_module.py | 26 +-- .../python/requirements.txt | 0 test_orc/python/src/test_orchestrator.py | 216 ++++++++++++++++++ 27 files changed, 320 insertions(+), 434 deletions(-) delete mode 100644 test-orchestrator/modules/base/bin/start_network_service delete mode 100644 test-orchestrator/modules/base/bin/wait_for_interface delete mode 100644 test-orchestrator/modules/template/conf/module_config.json delete mode 100644 test-orchestrator/python/src/docker_control.py delete mode 100644 test-orchestrator/python/src/test_orchestrator.py rename {test-orchestrator => test_orc}/modules/base/base.Dockerfile (92%) rename {test-orchestrator => test_orc}/modules/base/bin/capture (57%) rename {test-orchestrator => test_orc}/modules/base/bin/setup_binaries (100%) rename {test-orchestrator => test_orc}/modules/base/bin/start_grpc (70%) rename {test-orchestrator => test_orc}/modules/base/bin/start_module (84%) create mode 100644 test_orc/modules/base/bin/wait_for_interface rename {test-orchestrator => test_orc}/modules/base/conf/module_config.json (100%) rename {test-orchestrator => test_orc}/modules/base/python/requirements.txt (100%) rename {test-orchestrator => test_orc}/modules/base/python/src/grpc/start_server.py (100%) rename {test-orchestrator => test_orc}/modules/base/python/src/logger.py (100%) rename test-orchestrator/modules/template/template.Dockerfile => test_orc/modules/baseline/baseline.Dockerfile (54%) rename test-orchestrator/modules/template/bin/start_network_service => test_orc/modules/baseline/bin/start_test_module (68%) create mode 100644 test_orc/modules/baseline/conf/module_config.json rename {test-orchestrator/modules/template => test_orc/modules/baseline}/python/src/logger.py (93%) rename {test-orchestrator/modules/template => test_orc/modules/baseline}/python/src/run.py (96%) rename {test-orchestrator/modules/template => test_orc/modules/baseline}/python/src/test_module.py (59%) rename {test-orchestrator => test_orc}/python/requirements.txt (100%) create mode 100644 test_orc/python/src/test_orchestrator.py diff --git a/cmd/start b/cmd/start index 43a295338..0b3f21769 100755 --- a/cmd/start +++ b/cmd/start @@ -5,6 +5,12 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi +# Ensure that /var/run/netns folder exists +mkdir -p /var/run/netns + +# Clear up existing runtime files +rm -rf runtime + # Check if python modules exist. Install if not [ ! -d "venv" ] && cmd/install diff --git a/conf/system.json.example b/conf/system.json.example index f7212ceb3..2d4b737d0 100644 --- a/conf/system.json.example +++ b/conf/system.json.example @@ -3,11 +3,5 @@ "device_intf": "enx123456789123", "internet_intf": "enx123456789124" }, - "modules": [ - { - "name": "template", - "enabled": true - } - ], "log_level": "INFO" } \ No newline at end of file diff --git a/framework/run.py b/framework/run.py index ad7c038ee..2b3a3bdf5 100644 --- a/framework/run.py +++ b/framework/run.py @@ -1,5 +1,24 @@ """Starts Test Run.""" +import logger from testrun import TestRun -testrun = TestRun() +LOGGER = logger.get_logger('runner') + +class TestRunner: + + def __init__(self): + + LOGGER.info('Starting Test Run') + + testrun = TestRun() + + testrun.load_config() + + testrun.start_network() + + testrun.run_tests() + + testrun.stop_network() + +runner = TestRunner() diff --git a/framework/testrun.py b/framework/testrun.py index 26a6eb5cd..b6f2e07e2 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -11,7 +11,6 @@ import sys import json import signal -import time import logger # Locate parent directory @@ -23,11 +22,11 @@ sys.path.append(net_orc_dir) #Add test_orc to Python path -test_orc_dir = os.path.join(parent_dir, 'test-orchestrator', 'python', 'src') +test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') sys.path.append(test_orc_dir) import network_orchestrator as net_orc # pylint: disable=wrong-import-position -import test_orchestrator as test_orc +import test_orchestrator as test_orc # pylint: disable=wrong-import-position LOGGER = logger.get_logger('test_run') CONFIG_FILE = "conf/system.json" @@ -42,18 +41,13 @@ class TestRun: # pylint: disable=too-few-public-methods """ def __init__(self): - LOGGER.info("Starting Test Run") + + self._net_orc = net_orc.NetworkOrchestrator() + self._test_orc = test_orc.TestOrchestrator() # Catch any exit signals self._register_exits() - self._start_network() - - # Keep application running - time.sleep(RUNTIME) - - self._stop_network() - def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) signal.signal(signal.SIGTERM, self._exit_handler) @@ -64,9 +58,9 @@ def _exit_handler(self, signum, arg): # pylint: disable=unused-argument LOGGER.debug("Exit signal received: " + str(signum)) if signum in (2, signal.SIGTERM): LOGGER.info("Exit signal received.") - self._stop_network() + self.stop_network() - def _load_config(self): + def load_config(self): """Loads all settings from the config file into memory.""" if not os.path.isfile(CONFIG_FILE): LOGGER.error("Configuration file is not present at " + CONFIG_FILE) @@ -78,22 +72,15 @@ def _load_config(self): self._net_orc.import_config(config_json) self._test_orc.import_config(config_json) - def _start_network(self): - # Create an instance of the network orchestrator - self._net_orc = net_orc.NetworkOrchestrator() - - # Create an instance of the test orchestrator - self._test_orc=test_orc.TestOrchestrator() - - # Load config file and pass to other components - self._load_config() + def start_network(self): + """Starts the network orchestrator and network services.""" # Load and build any unbuilt network containers self._net_orc.load_network_modules() self._net_orc.build_network_modules() - # Load and build any unbuilt test module containers - self._test_orc.build_modules() + self._net_orc.stop_networking_services(kill=True) + self._net_orc.restore_net() # Create baseline network self._net_orc.create_net() @@ -103,16 +90,18 @@ def _start_network(self): LOGGER.info("Network is ready.") - # Begin testing - self._test_orc.run_test_modules() + def run_tests(self): + """Iterate through and start all test modules.""" - # Shutdown after testing completed - self._stop_network() + self._test_orc.load_test_modules() + self._test_orc.build_test_modules() + + # Begin testing + self._test_orc.run_test_modules() - def _stop_network(self): - LOGGER.info("Stopping Test Run") + def stop_network(self): + """Commands the net_orc to stop the network and clean up.""" self._net_orc.stop_networking_services(kill=True) - self._test_orc.stop_modules(kill=True) self._net_orc.restore_net() sys.exit(0) diff --git a/test-orchestrator/modules/base/bin/start_network_service b/test-orchestrator/modules/base/bin/start_network_service deleted file mode 100644 index 9f2336afa..000000000 --- a/test-orchestrator/modules/base/bin/start_network_service +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Place holder function for testing and validation -# Each network module should include a start_networkig_service -# file that overwrites this one to boot all of the its specific -# requirements to run. - -echo "Starting Network Service..." -echo " This is not a real network service, just a test" -echo "Network Service Started" - -# Do Nothing, just keep the module alive -while true;do sleep 1; done \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/wait_for_interface b/test-orchestrator/modules/base/bin/wait_for_interface deleted file mode 100644 index 9135edf14..000000000 --- a/test-orchestrator/modules/base/bin/wait_for_interface +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Default interface should be veth0 for all containers -DEFAULT_IFACE=veth0 - -# Allow a user to define an interface by passing it into this script -DEFINED_IFACE=$1 - -# Select which interace to use -if [[ -z $DEFINED_IFACE ]] -then - INTF=$DEFAULT_IFACE -else - INTF=$DEFINED_IFACE -fi - - -# Wait for local interface to be ready -while ! ip link show $INTF; do - echo $INTF is not yet ready. Waiting 3 seconds - sleep 3 -done \ No newline at end of file diff --git a/test-orchestrator/modules/template/conf/module_config.json b/test-orchestrator/modules/template/conf/module_config.json deleted file mode 100644 index 6b276ef6d..000000000 --- a/test-orchestrator/modules/template/conf/module_config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "config": { - "meta": { - "name": "template", - "display_name": "Template", - "description": "Template for building network service modules" - }, - "network": { - "interface": "eth0", - "enable_wan": false, - "ip_index": 9 - }, - "grpc": { - "port": 50001 - }, - "docker": { - "enable_container": true, - "mounts": [ - { - "source": "runtime/testing", - "target": "/runtime/testing" - } - ] - } - } -} \ No newline at end of file diff --git a/test-orchestrator/python/src/docker_control.py b/test-orchestrator/python/src/docker_control.py deleted file mode 100644 index 25061b064..000000000 --- a/test-orchestrator/python/src/docker_control.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 - -import logger -import docker -import os -import json -from docker.types import Mount - -LOG_NAME = "docker_cntl" -LOGGER = None -MODULES_DIR = "modules" -MODULE_CONFIG = "conf/module_config.json" - - -class DockerControl: - - def __init__(self,module): - self._modules = [] - - #Resolve the path to the test-orchestrator folder - self._path = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.realpath(__file__)))) - - #Resolve the path to the test-run folder - self._root_path = os.path.abspath(os.path.join(self._path,os.pardir)) - - self.add_logger(module) - - def add_logger(self, module): - global LOGGER - LOGGER = logger.get_logger(LOG_NAME, module) - - def _build_modules(self): - LOGGER.info("Building docker images...") - for module in self._modules: - self._build_module(module) - - def _build_module(self, module): - LOGGER.debug("Building docker image for module " + module.dir_name) - client = docker.from_env() - client.images.build( - dockerfile=os.path.join(module.dir, module.build_file), - path=self._path, - forcerm=True,# Cleans up intermediate containers during build - tag=module.image_name - ) - - def _get_module(self,name): - for module in self._modules: - if name == module.name: - return module - return None - - def _get_module_status(self,module): - container = self._get_module_container(module) - if container is not None: - return container.status - return None - - def _get_module_container(self, module): - LOGGER.debug("Resolving test module container: " + - module.container_name) - container = None - try: - client = docker.from_env() - container = client.containers.get(module.container_name) - except docker.errors.NotFound: - LOGGER.debug("Container " + - module.container_name + " not found") - except Exception as e: - LOGGER.error("Failed to resolve container") - LOGGER.error(e) - return container - - def _start_modules(self): - LOGGER.info("Starting test modules") - for module in self._modules: - # Test modules may just be Docker images, so we do not want to start them as containers - if not module.enable_container: - continue - - self._start_module(module) - - LOGGER.info("All network services are running") - - def _start_module(self, module): - - LOGGER.debug("Starting test module " + module.display_name) - try: - client = docker.from_env() - module.container = client.containers.run( - module.image_name, - auto_remove=True, - cap_add=["NET_ADMIN"], - name=module.container_name, - hostname=module.container_name, - privileged=True, - detach=True, - mounts=module.mounts, - environment={"HOST_USER": os.getlogin()} - ) - except Exception as client_error: - LOGGER.error("Container run error") - LOGGER.error(client_error) - - def _stop_modules(self, kill=False): - LOGGER.info("Stopping test modules") - for module in self._modules: - # Test modules may just be Docker images, so we do not want to stop them - if not module.enable_container: - continue - self._stop_module(module, kill) - - def _stop_module(self, module, kill=False): - LOGGER.debug("Stopping test module " + module.container_name) - try: - container = self._get_module_container(module) - if container is not None: - if kill: - LOGGER.debug("Killing container:" + - module.container_name) - container.kill() - else: - LOGGER.debug("Stopping container:" + - module.container_name) - container.stop() - LOGGER.debug("Container stopped:" + module.container_name) - except Exception as error: - LOGGER.error("Container stop error") - LOGGER.error(error) - - - def load_modules(self): - modules_dir = os.path.join(self._path, MODULES_DIR) - LOGGER.debug("Loading modules from /" + modules_dir) - - loaded_modules = "Loaded the following test modules: " - for module_dir in os.listdir(modules_dir): - - LOGGER.debug("Loading Module from: " + module_dir) - # Load basic module information - module = Module() - module_json = json.load(open(os.path.join( - self._path, modules_dir, module_dir, MODULE_CONFIG), encoding='UTF-8')) - LOGGER.debug(module_json) - module.name = module_json['config']['meta']['name'] - module.display_name = module_json['config']['meta']['display_name'] - module.description = module_json['config']['meta']['description'] - module.dir = os.path.join( - self._path, modules_dir, module_dir) - module.dir_name = module_dir - module.build_file = module_dir + ".Dockerfile" - module.container_name = "tr-ct-" + module.dir_name + "-test" - module.image_name = "test-run/" + module.dir_name + "-test" - - # Attach folder mounts to network module - if "docker" in module_json['config']: - if "mounts" in module_json['config']['docker']: - for mount_point in module_json['config']['docker']['mounts']: - module.mounts.append(Mount( - target=mount_point['target'], - source=os.path.join( - self._root_path, mount_point['source']), - type='bind' - )) - - # Determine if this is a container or just an image/template - if "enable_container" in module_json['config']['docker']: - module.enable_container = module_json['config']['docker']['enable_container'] - - self._modules.append(module) - - loaded_modules += module.dir_name + " " - - LOGGER.info(loaded_modules) - - -class Module: - - def __init__(self): - self.name = None - self.display_name = None - self.description = None - - self.container = None - self.container_name = None - self.image_name = None - - # Absolute path - self.dir = None - self.dir_name = None - self.build_file = None - self.mounts = [] - - self.enable_container = True diff --git a/test-orchestrator/python/src/test_orchestrator.py b/test-orchestrator/python/src/test_orchestrator.py deleted file mode 100644 index c64c691ea..000000000 --- a/test-orchestrator/python/src/test_orchestrator.py +++ /dev/null @@ -1,79 +0,0 @@ -from docker_control import DockerControl -import os -import logger -import json -import time -import shutil - -LOG_NAME = "test_orc" -LOGGER = None -RUNTIME_DIR = "runtime/testing" -TEST_MODULES_DIR = "tests/modules" -CONFIG_FILE = "conf/system.json" -MODULE_NAME = "test_orchestrator" - -class TestOrchestrator: - - def __init__(self): - self._test_modules = [] - self._module_config = None - - self._path = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.realpath(__file__)))) - - # Resolve the path to the test-run folder - self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) - - shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR)) - os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) - - self.add_logger(MODULE_NAME) - - self._docker_cntrl = DockerControl(MODULE_NAME) - - def add_logger(self,module): - global LOGGER - LOGGER = logger.get_logger(LOG_NAME, module) - - def run_test_modules(self): - LOGGER.info("Running test modules") - for module in self._test_modules: - if module["enabled"]: - self.run_test_module(module) - LOGGER.info("All tests complete") - - def run_test_module(self, module_config): - # Start the test container - LOGGER.info("Resolving container for test module: " + - module_config["name"]) - module = self._docker_cntrl._get_module(module_config["name"]) - if module is not None: - LOGGER.info("Test module container resolved: " + - module.container_name) - self._docker_cntrl._start_module(module) - - # Wait for the container to exit - status = self._docker_cntrl._get_module_status(module) - LOGGER.info("Test module " + module.display_name + " status: " + status) - if status == "running": - LOGGER.info("Waiting for test module " + module.display_name + " to complete") - while status == "running": - time.sleep(1) - status = self._docker_cntrl._get_module_status(module) - LOGGER.info("Test module " + module.display_name + " done") - - def start_modules(self): - self._docker_cntrl._start_modules() - - def stop_modules(self, kill=False): - self._docker_cntrl._stop_modules(kill=kill) - - def build_modules(self): - # Load and build any unbuilt network containers - self._docker_cntrl.load_modules() - self._docker_cntrl._build_modules() - - def import_config(self, json_config): - modules = json_config['modules'] - for module in modules: - self._test_modules.append(module) diff --git a/test-orchestrator/modules/base/base.Dockerfile b/test_orc/modules/base/base.Dockerfile similarity index 92% rename from test-orchestrator/modules/base/base.Dockerfile rename to test_orc/modules/base/base.Dockerfile index 7edbb1bbd..a4d22cae2 100644 --- a/test-orchestrator/modules/base/base.Dockerfile +++ b/test_orc/modules/base/base.Dockerfile @@ -19,5 +19,5 @@ RUN dos2unix /testrun/bin/* # Make sure all the bin files are executable RUN chmod u+x /testrun/bin/* -#Start the network module +# Start the test module ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/capture b/test_orc/modules/base/bin/capture similarity index 57% rename from test-orchestrator/modules/base/bin/capture rename to test_orc/modules/base/bin/capture index 45c627040..dccafb0c5 100644 --- a/test-orchestrator/modules/base/bin/capture +++ b/test_orc/modules/base/bin/capture @@ -4,22 +4,11 @@ MODULE_NAME=$1 # Define the local file location for the capture to be saved -PCAP_DIR="/runtime/testing/" +PCAP_DIR="/runtime/output/" PCAP_FILE=$MODULE_NAME.pcap -# Default interface should be veth0 for all containers -DEFAULT_IFACE=veth0 - # Allow a user to define an interface by passing it into this script -DEFINED_IFACE=$2 - -# Select which interace to use -if [[ -z $DEFINED_IFACE ]] -then - INTERFACE=$DEFAULT_IFACE -else - INTERFACE=$DEFINED_IFACE -fi +INTERFACE=$2 # Create the output directory and start the capture mkdir -p $PCAP_DIR @@ -27,5 +16,5 @@ chown $HOST_USER:$HOST_USER $PCAP_DIR echo "PCAP Dir: $PCAP_DIR/$PCAP_FILE" tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & -#Small pause to let the capture to start +# Small pause to let the capture to start sleep 1 \ No newline at end of file diff --git a/test-orchestrator/modules/base/bin/setup_binaries b/test_orc/modules/base/bin/setup_binaries similarity index 100% rename from test-orchestrator/modules/base/bin/setup_binaries rename to test_orc/modules/base/bin/setup_binaries diff --git a/test-orchestrator/modules/base/bin/start_grpc b/test_orc/modules/base/bin/start_grpc similarity index 70% rename from test-orchestrator/modules/base/bin/start_grpc rename to test_orc/modules/base/bin/start_grpc index 9792b4bd4..917381e89 100644 --- a/test-orchestrator/modules/base/bin/start_grpc +++ b/test_orc/modules/base/bin/start_grpc @@ -4,14 +4,14 @@ GRPC_DIR="/testrun/python/src/grpc" GRPC_PROTO_DIR="proto" GRPC_PROTO_FILE="grpc.proto" -#Move into the grpc directory +# Move into the grpc directory pushd $GRPC_DIR >/dev/null 2>&1 -#Build the grpc proto file every time before starting server +# Build the grpc proto file every time before starting server python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. popd >/dev/null 2>&1 -#Start the grpc server +# Start the grpc server python3 -u $GRPC_DIR/start_server.py $@ diff --git a/test-orchestrator/modules/base/bin/start_module b/test_orc/modules/base/bin/start_module similarity index 84% rename from test-orchestrator/modules/base/bin/start_module rename to test_orc/modules/base/bin/start_module index 7fdcbc404..a9f5402f4 100644 --- a/test-orchestrator/modules/base/bin/start_module +++ b/test_orc/modules/base/bin/start_module @@ -15,7 +15,7 @@ useradd $HOST_USER sysctl net.ipv6.conf.all.disable_ipv6=0 sysctl -p -#Read in the config file +# Read in the config file CONF_FILE="/testrun/conf/module_config.json" CONF=`cat $CONF_FILE` @@ -46,16 +46,13 @@ else INTF=$DEFINED_IFACE fi -echo "Starting module $MODULE_NAME on local interface $INTF..." +echo "Starting module $MODULE_NAME..." $BIN_DIR/setup_binaries $BIN_DIR # Wait for interface to become ready $BIN_DIR/wait_for_interface $INTF -# Small pause to let the interface stabalize before starting the capture -#sleep 1 - # Start network capture $BIN_DIR/capture $MODULE_NAME $INTF @@ -72,8 +69,8 @@ then fi fi -#Small pause to let all core services stabalize +# Small pause to let all core services stabalize sleep 3 -#Start the networking service -$BIN_DIR/start_network_service $MODULE_NAME $INTF \ No newline at end of file +# Start the networking service +$BIN_DIR/start_test_module $MODULE_NAME $INTF \ No newline at end of file diff --git a/test_orc/modules/base/bin/wait_for_interface b/test_orc/modules/base/bin/wait_for_interface new file mode 100644 index 000000000..c9c1682f0 --- /dev/null +++ b/test_orc/modules/base/bin/wait_for_interface @@ -0,0 +1,10 @@ +#!/bin/bash + +# Allow a user to define an interface by passing it into this script +INTF=$1 + +# Wait for local interface to be ready +while ! ip link show $INTF; do + echo $INTF is not yet ready. Waiting 3 seconds + sleep 3 +done \ No newline at end of file diff --git a/test-orchestrator/modules/base/conf/module_config.json b/test_orc/modules/base/conf/module_config.json similarity index 100% rename from test-orchestrator/modules/base/conf/module_config.json rename to test_orc/modules/base/conf/module_config.json diff --git a/test-orchestrator/modules/base/python/requirements.txt b/test_orc/modules/base/python/requirements.txt similarity index 100% rename from test-orchestrator/modules/base/python/requirements.txt rename to test_orc/modules/base/python/requirements.txt diff --git a/test-orchestrator/modules/base/python/src/grpc/start_server.py b/test_orc/modules/base/python/src/grpc/start_server.py similarity index 100% rename from test-orchestrator/modules/base/python/src/grpc/start_server.py rename to test_orc/modules/base/python/src/grpc/start_server.py diff --git a/test-orchestrator/modules/base/python/src/logger.py b/test_orc/modules/base/python/src/logger.py similarity index 100% rename from test-orchestrator/modules/base/python/src/logger.py rename to test_orc/modules/base/python/src/logger.py diff --git a/test-orchestrator/modules/template/template.Dockerfile b/test_orc/modules/baseline/baseline.Dockerfile similarity index 54% rename from test-orchestrator/modules/template/template.Dockerfile rename to test_orc/modules/baseline/baseline.Dockerfile index bdbb79c41..68e28d12c 100644 --- a/test-orchestrator/modules/template/template.Dockerfile +++ b/test_orc/modules/baseline/baseline.Dockerfile @@ -2,10 +2,10 @@ FROM test-run/base-test:latest # Copy over all configuration files -COPY modules/template/conf /testrun/conf +COPY modules/baseline/conf /testrun/conf # Load device binary files -COPY modules/template/bin /testrun/bin +COPY modules/baseline/bin /testrun/bin # Copy over all python files -COPY modules/template/python /testrun/python \ No newline at end of file +COPY modules/baseline/python /testrun/python \ No newline at end of file diff --git a/test-orchestrator/modules/template/bin/start_network_service b/test_orc/modules/baseline/bin/start_test_module similarity index 68% rename from test-orchestrator/modules/template/bin/start_network_service rename to test_orc/modules/baseline/bin/start_test_module index 3e1053700..f8b1ca463 100644 --- a/test-orchestrator/modules/template/bin/start_network_service +++ b/test_orc/modules/baseline/bin/start_test_module @@ -2,13 +2,10 @@ # An example startup script that does the bare minimum to start # a test module via a pyhon script. Each test module should include a -# start_network_service file that overwrites this one to boot all of its -#specific requirements to run. +# start_test_module file that overwrites this one to boot all of its +# specific requirements to run. -echo "Starting Network Service..." -echo " This is not a real network service, just a test" - -#Define where all the python source files are located +# Define where the python source files are located PYTHON_SRC_DIR=/testrun/python/src # Fetch module name @@ -23,14 +20,14 @@ DEFINED_IFACE=$2 # Select which interace to use if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] then - echo "No Interface Defined, defaulting to veth0" + echo "No interface defined, defaulting to veth0" INTF=$DEFAULT_IFACE else INTF=$DEFINED_IFACE fi #Create and set permissions on the log files -LOG_FILE=/runtime/network/$MODULE_NAME.log +LOG_FILE=/runtime/output/$MODULE_NAME.log touch $LOG_FILE chown $HOST_USER:$HOST_USER $LOG_FILE diff --git a/test_orc/modules/baseline/conf/module_config.json b/test_orc/modules/baseline/conf/module_config.json new file mode 100644 index 000000000..1b8b7b9ba --- /dev/null +++ b/test_orc/modules/baseline/conf/module_config.json @@ -0,0 +1,21 @@ +{ + "config": { + "meta": { + "name": "baseline", + "display_name": "Baseline", + "description": "Baseline test" + }, + "network": { + "interface": "eth0", + "enable_wan": false, + "ip_index": 9 + }, + "grpc": { + "port": 50001 + }, + "docker": { + "enable_container": true, + "timeout": 30 + } + } +} \ No newline at end of file diff --git a/test-orchestrator/modules/template/python/src/logger.py b/test_orc/modules/baseline/python/src/logger.py similarity index 93% rename from test-orchestrator/modules/template/python/src/logger.py rename to test_orc/modules/baseline/python/src/logger.py index 7294b77fd..641aa16b4 100644 --- a/test-orchestrator/modules/template/python/src/logger.py +++ b/test_orc/modules/baseline/python/src/logger.py @@ -3,7 +3,6 @@ import json import logging import os -import sys LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" @@ -11,7 +10,7 @@ _DEFAULT_LEVEL = logging.INFO _CONF_DIR = "conf" _CONF_FILE_NAME = "system.json" -_LOG_DIR = "/runtime/testing/" +_LOG_DIR = "/runtime/output/" # Set log level try: @@ -25,7 +24,6 @@ log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) - def add_file_handler(log, logFile): handler = logging.FileHandler(_LOG_DIR+logFile+".log") handler.setFormatter(log_format) diff --git a/test-orchestrator/modules/template/python/src/run.py b/test_orc/modules/baseline/python/src/run.py similarity index 96% rename from test-orchestrator/modules/template/python/src/run.py rename to test_orc/modules/baseline/python/src/run.py index dcf2a9861..7ff11559f 100644 --- a/test-orchestrator/modules/template/python/src/run.py +++ b/test_orc/modules/baseline/python/src/run.py @@ -3,7 +3,6 @@ import argparse import signal import sys -import time import logger from test_module import TestModule diff --git a/test-orchestrator/modules/template/python/src/test_module.py b/test_orc/modules/baseline/python/src/test_module.py similarity index 59% rename from test-orchestrator/modules/template/python/src/test_module.py rename to test_orc/modules/baseline/python/src/test_module.py index f5bbf9488..2e4ce0da2 100644 --- a/test-orchestrator/modules/template/python/src/test_module.py +++ b/test_orc/modules/baseline/python/src/test_module.py @@ -1,32 +1,20 @@ #!/usr/bin/env python3 -import signal import time -import sys -import argparse import logger -LOGGER = None -LOG_NAME = "test_module_template" +LOG_NAME = "test_baseline" +LOGGER = logger.get_logger(LOG_NAME) + class TestModule: - def __init__(self, module): + def __init__(self): self.module_test1 = None self.module_test2 = None self.module_test3 = None - self.add_logger(module) - - signal.signal(signal.SIGINT, self.handler) - signal.signal(signal.SIGTERM, self.handler) - signal.signal(signal.SIGABRT, self.handler) - signal.signal(signal.SIGQUIT, self.handler) - - def add_logger(self, module): - global LOGGER - LOGGER = logger.get_logger(LOG_NAME, module) # Make up some fake test results def run_tests(self): @@ -38,6 +26,8 @@ def run_tests(self): self.module_test2 = False LOGGER.info("Test 2 complete.") + time.sleep(10) + def generate_results(self): self.print_test_result("Test 1", self.module_test1) self.print_test_result("Test 2", self.module_test2) @@ -49,7 +39,3 @@ def print_test_result(self, test_name, result): test_name + ": Pass" if result else test_name + ": Fail") else: LOGGER.info(test_name + " Skipped") - - def handler(self, signum, frame): - if (signum == 2 or signal == signal.SIGTERM): - exit(1) diff --git a/test-orchestrator/python/requirements.txt b/test_orc/python/requirements.txt similarity index 100% rename from test-orchestrator/python/requirements.txt rename to test_orc/python/requirements.txt diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py new file mode 100644 index 000000000..20227765e --- /dev/null +++ b/test_orc/python/src/test_orchestrator.py @@ -0,0 +1,216 @@ +import os +import json +import time +import shutil +import docker +from docker.types import Mount +import logger + +LOG_NAME = "test_orc" +LOGGER = logger.get_logger('test_orc') +RUNTIME_DIR = "runtime" +TEST_MODULES_DIR = "modules" +MODULE_CONFIG = "conf/module_config.json" + +class TestOrchestrator: + """Manages and controls the test modules.""" + + def __init__(self): + self._test_modules = [] + self._module_config = None + + self._path = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__)))) + + # Resolve the path to the test-run folder + self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) + + shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR), ignore_errors=True) + os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) + + def import_config(self, json_config): + """Load settings from JSON object into memory.""" + + # No relevant config options in system.json as of yet + + def get_test_module(self, name): + """Returns a test module by the module name.""" + for module in self._test_modules: + if name == module.name: + return module + return None + + def run_test_modules(self): + """Iterates through each test module and starts the container.""" + LOGGER.info("Running test modules...") + for module in self._test_modules: + self.run_test_module(module) + LOGGER.info("All tests complete") + + def run_test_module(self, module): + """Start the test container and extract the results.""" + + if module is None or not module.enable_container: + return + + LOGGER.info("Running test module " + module.display_name) + try: + + container_runtime_dir = os.path.join(self._root_path, "runtime/test/" + module.name) + os.makedirs(container_runtime_dir) + + client = docker.from_env() + + module.container = client.containers.run( + module.image_name, + auto_remove=True, + cap_add=["NET_ADMIN"], + name=module.container_name, + hostname=module.container_name, + privileged=True, + detach=True, + mounts=[Mount( + target="/runtime/output", + source=container_runtime_dir, + type='bind' + )], + environment={"HOST_USER": os.getlogin()} + ) + except (docker.errors.APIError, docker.errors.ContainerError) as container_error: + LOGGER.error("Container run error") + LOGGER.error(container_error) + + # Determine the module timeout time + test_module_timeout = time.time() + module.timeout + status = self._get_module_status(module) + + while time.time() < test_module_timeout or status == 'exited': + time.sleep(1) + status = self._get_module_status(module) + + LOGGER.info("Test module " + module.display_name + " has finished") + + def _get_module_status(self, module): + return self._get_module_container(module.container.name).status + + def _get_module_container(self, module): + container = None + try: + client = docker.from_env() + container = client.containers.get(module.container_name) + except docker.errors.NotFound: + LOGGER.debug("Container " + + module.container_name + " not found") + except docker.errors.APIError as error: + LOGGER.error("Failed to resolve container") + LOGGER.error(error) + return container + + def load_test_modules(self): + """Import module configuration from module_config.json.""" + + modules_dir = os.path.join(self._path, TEST_MODULES_DIR) + + LOGGER.debug("Loading test modules from /" + modules_dir) + loaded_modules = "Loaded the following test modules: " + + for module_dir in os.listdir(modules_dir): + + LOGGER.debug("Loading module from: " + module_dir) + + # Load basic module information + module = TestModule() + module_json = json.load(open(os.path.join( + self._path, modules_dir, module_dir, MODULE_CONFIG), encoding='UTF-8')) + module.name = module_json['config']['meta']['name'] + module.display_name = module_json['config']['meta']['display_name'] + module.description = module_json['config']['meta']['description'] + module.dir = os.path.join( + self._path, modules_dir, module_dir) + module.dir_name = module_dir + module.build_file = module_dir + ".Dockerfile" + module.container_name = "tr-ct-" + module.dir_name + "-test" + module.image_name = "test-run/" + module.dir_name + "-test" + + if 'timeout' in module_json['config']['docker']: + module.timeout = module_json['config']['docker']['timeout'] + + # Determine if this is a container or just an image/template + if "enable_container" in module_json['config']['docker']: + module.enable_container = module_json['config']['docker']['enable_container'] + + self._test_modules.append(module) + + loaded_modules += module.dir_name + " " + + LOGGER.info(loaded_modules) + + def build_test_modules(self): + """Build all test modules.""" + LOGGER.info("Building test modules...") + for module in self._test_modules: + self._build_test_module(module) + + def _build_test_module(self, module): + LOGGER.debug("Building docker image for module " + module.dir_name) + client = docker.from_env() + try: + client.images.build( + dockerfile=os.path.join(module.dir, module.build_file), + path=self._path, + forcerm=True, # Cleans up intermediate containers during build + tag=module.image_name + ) + except docker.errors.BuildError as error: + LOGGER.error(error) + + def _stop_modules(self, kill=False): + LOGGER.debug("Stopping test modules") + for module in self._test_modules: + # Test modules may just be Docker images, so we do not want to stop them + if not module.enable_container: + continue + self._stop_module(module, kill) + + def _stop_module(self, module, kill=False): + LOGGER.debug("Stopping test module " + module.container_name) + try: + container = module.container + if container is not None: + if kill: + LOGGER.debug("Killing container:" + + module.container_name) + container.kill() + else: + LOGGER.debug("Stopping container:" + + module.container_name) + container.stop() + LOGGER.debug("Container stopped:" + module.container_name) + except Exception as error: + LOGGER.error("Container stop error") + LOGGER.error(error) + + def _get_module_status(self, module): + if module.container is not None: + return module.container.status + return None + +class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes + """Represents a test module.""" + + def __init__(self): + self.name = None + self.display_name = None + self.description = None + + self.build_file = None + self.container = None + self.container_name = None + self.image_name = None + self.enable_container = True + + self.timeout = 60 + + # Absolute path + self.dir = None + self.dir_name = None From e8498114d00774da72f4313bdb1639849a8ed8f9 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 24 Apr 2023 11:16:46 -0600 Subject: [PATCH 09/20] Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator --- cmd/start | 44 ++++++++++++++--------------- framework/run.py | 67 ++++++++++++++++++++++++++++---------------- framework/testrun.py | 29 +++++++++++++------ 3 files changed, 86 insertions(+), 54 deletions(-) diff --git a/cmd/start b/cmd/start index 0b3f21769..257fb2511 100755 --- a/cmd/start +++ b/cmd/start @@ -1,23 +1,23 @@ -#!/bin/bash -e - -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo cmd/start" - exit 1 -fi - -# Ensure that /var/run/netns folder exists -mkdir -p /var/run/netns - -# Clear up existing runtime files -rm -rf runtime - -# Check if python modules exist. Install if not -[ ! -d "venv" ] && cmd/install - -# Activate Python virtual environment -source venv/bin/activate - -# TODO: Execute python code -python -u framework/run.py - +#!/bin/bash -e + +if [[ "$EUID" -ne 0 ]]; then + echo "Must run as root. Use sudo cmd/start" + exit 1 +fi + +# Ensure that /var/run/netns folder exists +mkdir -p /var/run/netns + +# Clear up existing runtime files +rm -rf runtime + +# Check if python modules exist. Install if not +[ ! -d "venv" ] && cmd/install + +# Activate Python virtual environment +source venv/bin/activate + +# TODO: Execute python code +python -u framework/run.py $@ + deactivate \ No newline at end of file diff --git a/framework/run.py b/framework/run.py index 2b3a3bdf5..36bdb004f 100644 --- a/framework/run.py +++ b/framework/run.py @@ -1,24 +1,43 @@ -"""Starts Test Run.""" - -import logger -from testrun import TestRun - -LOGGER = logger.get_logger('runner') - -class TestRunner: - - def __init__(self): - - LOGGER.info('Starting Test Run') - - testrun = TestRun() - - testrun.load_config() - - testrun.start_network() - - testrun.run_tests() - - testrun.stop_network() - -runner = TestRunner() +"""Starts Test Run.""" + +import logger +from testrun import TestRun +import argparse +import sys + +LOGGER = logger.get_logger('runner') + + +class TestRunner: + + def __init__(self,local_net=True): + + LOGGER.info('Starting Test Run') + + testrun = TestRun(local_net) + + testrun.load_config() + + testrun.start_network() + + testrun.run_tests() + + testrun.stop_network() + + +def run(argv): + parser = argparse.ArgumentParser(description="Test Run", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-r", "--remote-net", action="store_false", + help='''Use the network orchestrator from the parent directory instead + of the one downloaded locally from the install script.''') + + args, unknown = parser.parse_known_args() + + print("local-net: " + str(args.remote_net)) + + runner = TestRunner(args.remote_net) + + +if __name__ == "__main__": + run(sys.argv) diff --git a/framework/testrun.py b/framework/testrun.py index b6f2e07e2..eccda5df6 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -17,15 +17,10 @@ current_dir = os.path.dirname(os.path.realpath(__file__)) parent_dir = os.path.dirname(current_dir) -# Add net_orc to Python path -net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') -sys.path.append(net_orc_dir) - #Add test_orc to Python path test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') sys.path.append(test_orc_dir) -import network_orchestrator as net_orc # pylint: disable=wrong-import-position import test_orchestrator as test_orc # pylint: disable=wrong-import-position LOGGER = logger.get_logger('test_run') @@ -40,13 +35,31 @@ class TestRun: # pylint: disable=too-few-public-methods orchestrator and user interface. """ - def __init__(self): + def __init__(self,local_net=True): + + # Catch any exit signals + self._register_exits() + + # Import the correct net orchestrator + self.import_orchestrators(local_net) self._net_orc = net_orc.NetworkOrchestrator() self._test_orc = test_orc.TestOrchestrator() - # Catch any exit signals - self._register_exits() + def import_orchestrators(self,local_net=True): + if local_net: + # Add local net_orc to Python path + net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') + else: + # Resolve the path to the test-run parent folder + root_dir = os.path.abspath(os.path.join(parent_dir, os.pardir)) + # Add manually cloned network orchestrator from parent folder + net_orc_dir = os.path.join(root_dir, 'network-orchestrator', 'python', 'src') + # Add net_orc to Python path + sys.path.append(net_orc_dir) + #Import the network orchestrator + global net_orc + import network_orchestrator as net_orc # pylint: disable=wrong-import-position def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) From f57a447d986d2e6a4d3dcbaaf4f9354e188ee281 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 24 Apr 2023 11:18:40 -0600 Subject: [PATCH 10/20] Fix baseline module Fix orchestrator exiting only after timeout --- .../modules/baseline/python/src/test_module.py | 7 ++++++- test_orc/python/src/test_orchestrator.py | 15 ++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test_orc/modules/baseline/python/src/test_module.py b/test_orc/modules/baseline/python/src/test_module.py index 2e4ce0da2..e37993740 100644 --- a/test_orc/modules/baseline/python/src/test_module.py +++ b/test_orc/modules/baseline/python/src/test_module.py @@ -10,11 +10,16 @@ class TestModule: - def __init__(self): + def __init__(self,module): self.module_test1 = None self.module_test2 = None self.module_test3 = None + self.add_logger(module) + + def add_logger(self, module): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, module) # Make up some fake test results def run_tests(self): diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 20227765e..8ea309946 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -83,15 +83,17 @@ def run_test_module(self, module): # Determine the module timeout time test_module_timeout = time.time() + module.timeout status = self._get_module_status(module) - - while time.time() < test_module_timeout or status == 'exited': + while time.time() < test_module_timeout and status == 'running': time.sleep(1) status = self._get_module_status(module) LOGGER.info("Test module " + module.display_name + " has finished") - def _get_module_status(self, module): - return self._get_module_container(module.container.name).status + def _get_module_status(self,module): + container = self._get_module_container(module) + if container is not None: + return container.status + return None def _get_module_container(self, module): container = None @@ -190,11 +192,6 @@ def _stop_module(self, module, kill=False): LOGGER.error("Container stop error") LOGGER.error(error) - def _get_module_status(self, module): - if module.container is not None: - return module.container.status - return None - class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Represents a test module.""" From 05f7dc3a76133cfe7e9a78c80e7c5188388699cd Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 24 Apr 2023 12:43:54 -0600 Subject: [PATCH 11/20] Add result file to baseline test module Change result format to match closer to design doc --- .../modules/baseline/bin/start_test_module | 3 ++ .../baseline/python/src/test_module.py | 40 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/test_orc/modules/baseline/bin/start_test_module b/test_orc/modules/baseline/bin/start_test_module index f8b1ca463..2ff3b1a1c 100644 --- a/test_orc/modules/baseline/bin/start_test_module +++ b/test_orc/modules/baseline/bin/start_test_module @@ -28,8 +28,11 @@ fi #Create and set permissions on the log files LOG_FILE=/runtime/output/$MODULE_NAME.log +RESULT_FILE=/runtime/output/$MODULE_NAME-result.json touch $LOG_FILE +touch $RESULT_FILE chown $HOST_USER:$HOST_USER $LOG_FILE +chown $HOST_USER:$HOST_USER $RESULT_FILE # Run the python scrip that will execute the tests for this module # -u flag allows python print statements diff --git a/test_orc/modules/baseline/python/src/test_module.py b/test_orc/modules/baseline/python/src/test_module.py index e37993740..30e87a2e5 100644 --- a/test_orc/modules/baseline/python/src/test_module.py +++ b/test_orc/modules/baseline/python/src/test_module.py @@ -2,19 +2,21 @@ import time import logger +import json LOG_NAME = "test_baseline" +RESULTS_DIR = "/runtime/output/" LOGGER = logger.get_logger(LOG_NAME) - class TestModule: - def __init__(self,module): + def __init__(self, module): self.module_test1 = None self.module_test2 = None self.module_test3 = None + self.module=module self.add_logger(module) def add_logger(self, module): @@ -34,13 +36,29 @@ def run_tests(self): time.sleep(10) def generate_results(self): - self.print_test_result("Test 1", self.module_test1) - self.print_test_result("Test 2", self.module_test2) - self.print_test_result("Test 3", self.module_test3) - - def print_test_result(self, test_name, result): - if result is not None: - LOGGER.info( - test_name + ": Pass" if result else test_name + ": Fail") + results = [] + results.append(self.generate_result("Test 1",self.module_test1)); + results.append(self.generate_result("Test 2",self.module_test2)); + results.append(self.generate_result("Test 3",self.module_test3)); + jsonResults = json.dumps({"results":results},indent=2) + self.write_results(jsonResults) + + def write_results(self,results): + RESULTS_FILE=RESULTS_DIR+self.module+"-result.json" + LOGGER.info("Writing results to " + RESULTS_FILE) + f = open(RESULTS_FILE, "w", encoding="utf-8") + f.write(results) + f.close() + + def generate_result(self, test_name, test_result): + if test_result is not None: + result = "compliant" if test_result else "non-compliant" else: - LOGGER.info(test_name + " Skipped") + result = "skipped" + LOGGER.info(test_name + ": " + result) + resDict = { + "name": test_name, + "result": result, + "description": "The device is " + result + } + return resDict From 9a3bdb4fd0496fc55bdcc3fba3f9d75f00af467e Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 25 Apr 2023 13:21:25 +0100 Subject: [PATCH 12/20] Refactor pylint --- cmd/start | 44 ++++++++++++++++++++++---------------------- framework/logger.py | 21 +++++++++------------ framework/run.py | 4 +--- framework/testrun.py | 6 +++--- 4 files changed, 35 insertions(+), 40 deletions(-) diff --git a/cmd/start b/cmd/start index 257fb2511..fa6bbc1e1 100755 --- a/cmd/start +++ b/cmd/start @@ -1,23 +1,23 @@ -#!/bin/bash -e - -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo cmd/start" - exit 1 -fi - -# Ensure that /var/run/netns folder exists -mkdir -p /var/run/netns - -# Clear up existing runtime files -rm -rf runtime - -# Check if python modules exist. Install if not -[ ! -d "venv" ] && cmd/install - -# Activate Python virtual environment -source venv/bin/activate - -# TODO: Execute python code -python -u framework/run.py $@ - +#!/bin/bash -e + +if [[ "$EUID" -ne 0 ]]; then + echo "Must run as root. Use sudo cmd/start" + exit 1 +fi + +# Ensure that /var/run/netns folder exists +mkdir -p /var/run/netns + +# Clear up existing runtime files +rm -rf runtime + +# Check if python modules exist. Install if not +[ ! -d "venv" ] && cmd/install + +# Activate Python virtual environment +source venv/bin/activate + +# TODO: Execute python code +python -u framework/run.py $@ + deactivate \ No newline at end of file diff --git a/framework/logger.py b/framework/logger.py index db4747af5..5df0cd954 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -3,7 +3,6 @@ import json import logging import os -import sys LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" @@ -15,33 +14,31 @@ # Set log level try: - system_conf_json = json.load( - open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) - log_level_str = system_conf_json['log_level'] - log_level = logging.getLevelName(log_level_str) + with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as system_conf_file: + system_conf_json = json.load(system_conf_file) + log_level_str = system_conf_json['log_level'] + log_level = logging.getLevelName(log_level_str) except: # TODO: Print out warning that log level is incorrect or missing log_level = _DEFAULT_LEVEL log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) - -def add_file_handler(log, logFile): - handler = logging.FileHandler(_LOG_DIR+logFile+".log") +def add_file_handler(log, log_file): + handler = logging.FileHandler(_LOG_DIR + log_file + ".log") handler.setFormatter(log_format) log.addHandler(handler) - def add_stream_handler(log): handler = logging.StreamHandler() handler.setFormatter(log_format) log.addHandler(handler) -def get_logger(name, logFile=None): +def get_logger(name, log_file=None): if name not in LOGGERS: LOGGERS[name] = logging.getLogger(name) LOGGERS[name].setLevel(log_level) add_stream_handler(LOGGERS[name]) - if logFile is not None: - add_file_handler(LOGGERS[name], logFile) + if log_file is not None: + add_file_handler(LOGGERS[name], log_file) return LOGGERS[name] diff --git a/framework/run.py b/framework/run.py index 36bdb004f..bd0919b36 100644 --- a/framework/run.py +++ b/framework/run.py @@ -10,7 +10,7 @@ class TestRunner: - def __init__(self,local_net=True): + def __init__(self, local_net=True): LOGGER.info('Starting Test Run') @@ -34,8 +34,6 @@ def run(argv): args, unknown = parser.parse_known_args() - print("local-net: " + str(args.remote_net)) - runner = TestRunner(args.remote_net) diff --git a/framework/testrun.py b/framework/testrun.py index eccda5df6..75590e9ce 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -46,7 +46,7 @@ def __init__(self,local_net=True): self._net_orc = net_orc.NetworkOrchestrator() self._test_orc = test_orc.TestOrchestrator() - def import_orchestrators(self,local_net=True): + def import_orchestrators(self,local_net=True): if local_net: # Add local net_orc to Python path net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') @@ -57,9 +57,9 @@ def import_orchestrators(self,local_net=True): net_orc_dir = os.path.join(root_dir, 'network-orchestrator', 'python', 'src') # Add net_orc to Python path sys.path.append(net_orc_dir) - #Import the network orchestrator + # Import the network orchestrator global net_orc - import network_orchestrator as net_orc # pylint: disable=wrong-import-position + import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) From 295650de5595e2e9ee8ba45a0f39248d3bd98ed3 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 25 Apr 2023 13:53:45 +0100 Subject: [PATCH 13/20] Skip test module if it failed to start --- framework/logger.py | 3 +-- framework/testrun.py | 13 +++++---- test_orc/modules/base/base.Dockerfile | 4 +-- test_orc/modules/base/python/src/logger.py | 14 +++++----- test_orc/modules/baseline/baseline.Dockerfile | 2 +- .../modules/baseline/bin/start_test_module | 2 +- .../baseline/python/src/test_module.py | 27 +++++++++---------- test_orc/python/src/test_orchestrator.py | 20 +++++++++----- 8 files changed, 44 insertions(+), 41 deletions(-) diff --git a/framework/logger.py b/framework/logger.py index 5df0cd954..6e96dfcd9 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""Manages stream and file loggers.""" import json import logging import os diff --git a/framework/testrun.py b/framework/testrun.py index 75590e9ce..22fa0295a 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -17,12 +17,6 @@ current_dir = os.path.dirname(os.path.realpath(__file__)) parent_dir = os.path.dirname(current_dir) -#Add test_orc to Python path -test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') -sys.path.append(test_orc_dir) - -import test_orchestrator as test_orc # pylint: disable=wrong-import-position - LOGGER = logger.get_logger('test_run') CONFIG_FILE = "conf/system.json" EXAMPLE_CONFIG_FILE = "conf/system.json.example" @@ -61,6 +55,12 @@ def import_orchestrators(self,local_net=True): global net_orc import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel + # Add test_orc to Python path + test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') + sys.path.append(test_orc_dir) + global test_orc + import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel + def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) signal.signal(signal.SIGTERM, self._exit_handler) @@ -107,7 +107,6 @@ def run_tests(self): """Iterate through and start all test modules.""" self._test_orc.load_test_modules() - self._test_orc.build_test_modules() # Begin testing diff --git a/test_orc/modules/base/base.Dockerfile b/test_orc/modules/base/base.Dockerfile index a4d22cae2..b5f35326a 100644 --- a/test_orc/modules/base/base.Dockerfile +++ b/test_orc/modules/base/base.Dockerfile @@ -1,10 +1,10 @@ -# Image name: test-run/base +# Image name: test-run/base-test FROM ubuntu:jammy # Install common software RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix -#Setup the base python requirements +# Setup the base python requirements COPY modules/base/python /testrun/python # Install all python requirements for the module diff --git a/test_orc/modules/base/python/src/logger.py b/test_orc/modules/base/python/src/logger.py index 4924512c6..0eb7b9ccf 100644 --- a/test_orc/modules/base/python/src/logger.py +++ b/test_orc/modules/base/python/src/logger.py @@ -15,7 +15,7 @@ # Set log level try: system_conf_json = json.load( - open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) + open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8')) log_level_str = system_conf_json['log_level'] log_level = logging.getLevelName(log_level_str) except: @@ -25,23 +25,21 @@ log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) -def add_file_handler(log, logFile): - handler = logging.FileHandler(_LOG_DIR+logFile+".log") +def add_file_handler(log, log_file): + handler = logging.FileHandler(_LOG_DIR+log_file+".log") handler.setFormatter(log_format) log.addHandler(handler) - def add_stream_handler(log): handler = logging.StreamHandler() handler.setFormatter(log_format) log.addHandler(handler) - -def get_logger(name, logFile=None): +def get_logger(name, log_file=None): if name not in LOGGERS: LOGGERS[name] = logging.getLogger(name) LOGGERS[name].setLevel(log_level) add_stream_handler(LOGGERS[name]) - if logFile is not None: - add_file_handler(LOGGERS[name], logFile) + if log_file is not None: + add_file_handler(LOGGERS[name], log_file) return LOGGERS[name] diff --git a/test_orc/modules/baseline/baseline.Dockerfile b/test_orc/modules/baseline/baseline.Dockerfile index 68e28d12c..5b634e6ee 100644 --- a/test_orc/modules/baseline/baseline.Dockerfile +++ b/test_orc/modules/baseline/baseline.Dockerfile @@ -1,4 +1,4 @@ -# Image name: test-run/dhcp-primary +# Image name: test-run/baseline-test FROM test-run/base-test:latest # Copy over all configuration files diff --git a/test_orc/modules/baseline/bin/start_test_module b/test_orc/modules/baseline/bin/start_test_module index 2ff3b1a1c..292b57de2 100644 --- a/test_orc/modules/baseline/bin/start_test_module +++ b/test_orc/modules/baseline/bin/start_test_module @@ -26,7 +26,7 @@ else INTF=$DEFINED_IFACE fi -#Create and set permissions on the log files +# Create and set permissions on the log files LOG_FILE=/runtime/output/$MODULE_NAME.log RESULT_FILE=/runtime/output/$MODULE_NAME-result.json touch $LOG_FILE diff --git a/test_orc/modules/baseline/python/src/test_module.py b/test_orc/modules/baseline/python/src/test_module.py index 30e87a2e5..440b87f7f 100644 --- a/test_orc/modules/baseline/python/src/test_module.py +++ b/test_orc/modules/baseline/python/src/test_module.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 +import json import time import logger -import json LOG_NAME = "test_baseline" RESULTS_DIR = "/runtime/output/" LOGGER = logger.get_logger(LOG_NAME) - class TestModule: def __init__(self, module): @@ -16,7 +15,7 @@ def __init__(self, module): self.module_test1 = None self.module_test2 = None self.module_test3 = None - self.module=module + self.module = module self.add_logger(module) def add_logger(self, module): @@ -37,28 +36,28 @@ def run_tests(self): def generate_results(self): results = [] - results.append(self.generate_result("Test 1",self.module_test1)); - results.append(self.generate_result("Test 2",self.module_test2)); - results.append(self.generate_result("Test 3",self.module_test3)); - jsonResults = json.dumps({"results":results},indent=2) - self.write_results(jsonResults) + results.append(self.generate_result("Test 1", self.module_test1)) + results.append(self.generate_result("Test 2", self.module_test2)) + results.append(self.generate_result("Test 3", self.module_test3)) + json_results = json.dumps({"results":results}, indent=2) + self.write_results(json_results) def write_results(self,results): - RESULTS_FILE=RESULTS_DIR+self.module+"-result.json" - LOGGER.info("Writing results to " + RESULTS_FILE) - f = open(RESULTS_FILE, "w", encoding="utf-8") + results_file=RESULTS_DIR+self.module+"-result.json" + LOGGER.info("Writing results to " + results_file) + f = open(results_file, "w", encoding="utf-8") f.write(results) f.close() - + def generate_result(self, test_name, test_result): if test_result is not None: result = "compliant" if test_result else "non-compliant" else: result = "skipped" LOGGER.info(test_name + ": " + result) - resDict = { + res_dict = { "name": test_name, "result": result, "description": "The device is " + result } - return resDict + return res_dict diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 8ea309946..396f533fa 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -1,3 +1,4 @@ +"""Provides high level management of the test orchestrator.""" import os import json import time @@ -77,12 +78,14 @@ def run_test_module(self, module): environment={"HOST_USER": os.getlogin()} ) except (docker.errors.APIError, docker.errors.ContainerError) as container_error: - LOGGER.error("Container run error") - LOGGER.error(container_error) + LOGGER.error("Test module " + module.display_name + " has failed to start") + LOGGER.debug(container_error) + return # Determine the module timeout time test_module_timeout = time.time() + module.timeout status = self._get_module_status(module) + while time.time() < test_module_timeout and status == 'running': time.sleep(1) status = self._get_module_status(module) @@ -122,13 +125,18 @@ def load_test_modules(self): # Load basic module information module = TestModule() - module_json = json.load(open(os.path.join( - self._path, modules_dir, module_dir, MODULE_CONFIG), encoding='UTF-8')) + with open(os.path.join( + self._path, + modules_dir, + module_dir, + MODULE_CONFIG), + encoding='UTF-8') as module_config_file: + module_json = json.load(module_config_file) + module.name = module_json['config']['meta']['name'] module.display_name = module_json['config']['meta']['display_name'] module.description = module_json['config']['meta']['description'] - module.dir = os.path.join( - self._path, modules_dir, module_dir) + module.dir = os.path.join(self._path, modules_dir, module_dir) module.dir_name = module_dir module.build_file = module_dir + ".Dockerfile" module.container_name = "tr-ct-" + module.dir_name + "-test" From 00ce66082549ea38c430ab2b8ebed6ab757aa4b8 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 25 Apr 2023 15:29:55 +0100 Subject: [PATCH 14/20] Refactor --- framework/logger.py | 15 ++++++++------- framework/run.py | 9 ++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/framework/logger.py b/framework/logger.py index 6e96dfcd9..7a9a71b2d 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -6,7 +6,7 @@ LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" _DATE_FORMAT = '%b %02d %H:%M:%S' -_DEFAULT_LEVEL = logging.INFO +_LOG_LEVEL = logging.INFO _CONF_DIR = "conf" _CONF_FILE_NAME = "system.json" _LOG_DIR = "runtime/testing/" @@ -15,11 +15,12 @@ try: with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as system_conf_file: system_conf_json = json.load(system_conf_file) - log_level_str = system_conf_json['log_level'] - log_level = logging.getLevelName(log_level_str) -except: - # TODO: Print out warning that log level is incorrect or missing - log_level = _DEFAULT_LEVEL + log_level_str = system_conf_json['log_level'] + _LOG_LEVEL = logging.getLevelName(log_level_str) +except Exception as error: + print(error) + # Do nothing as fallback log level will be used + pass log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) @@ -36,7 +37,7 @@ def add_stream_handler(log): def get_logger(name, log_file=None): if name not in LOGGERS: LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(log_level) + LOGGERS[name].setLevel(_LOG_LEVEL) add_stream_handler(LOGGERS[name]) if log_file is not None: add_file_handler(LOGGERS[name], log_file) diff --git a/framework/run.py b/framework/run.py index bd0919b36..d2643d956 100644 --- a/framework/run.py +++ b/framework/run.py @@ -1,13 +1,12 @@ """Starts Test Run.""" -import logger -from testrun import TestRun import argparse import sys +from testrun import TestRun +import logger LOGGER = logger.get_logger('runner') - class TestRunner: def __init__(self, local_net=True): @@ -29,12 +28,12 @@ def run(argv): parser = argparse.ArgumentParser(description="Test Run", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-r", "--remote-net", action="store_false", - help='''Use the network orchestrator from the parent directory instead + help='''Use the network orchestrator from the parent directory instead of the one downloaded locally from the install script.''') args, unknown = parser.parse_known_args() - runner = TestRunner(args.remote_net) + TestRunner(args.remote_net) if __name__ == "__main__": From da916e3c6d96d715826eded485faf7c5bf492464 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 25 Apr 2023 15:43:19 +0100 Subject: [PATCH 15/20] Check for valid log level --- framework/logger.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/framework/logger.py b/framework/logger.py index 7a9a71b2d..64d8fdb97 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -6,21 +6,25 @@ LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" _DATE_FORMAT = '%b %02d %H:%M:%S' +_DEFAULT_LOG_LEVEL = logging.INFO _LOG_LEVEL = logging.INFO _CONF_DIR = "conf" _CONF_FILE_NAME = "system.json" _LOG_DIR = "runtime/testing/" # Set log level +with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as system_conf_file: + system_conf_json = json.load(system_conf_file) +log_level_str = system_conf_json['log_level'] + +temp_log = logging.getLogger('temp') try: - with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as system_conf_file: - system_conf_json = json.load(system_conf_file) - log_level_str = system_conf_json['log_level'] + temp_log.setLevel(logging.getLevelName(log_level_str)) _LOG_LEVEL = logging.getLevelName(log_level_str) -except Exception as error: - print(error) - # Do nothing as fallback log level will be used - pass +except ValueError: + print('Invalid log level set in ' + _CONF_DIR + '/' + _CONF_FILE_NAME + + '. Using INFO as log level') + _LOG_LEVEL = _DEFAULT_LOG_LEVEL log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) From e21a7571801d0b99ab38294162e33ca1c321b6b0 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 26 Apr 2023 09:55:31 -0600 Subject: [PATCH 16/20] Add config file arg Misc changes to network start procedure --- framework/run.py | 10 ++++--- framework/testrun.py | 65 ++++++++++++++++++++------------------------ 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/framework/run.py b/framework/run.py index d2643d956..7be301da5 100644 --- a/framework/run.py +++ b/framework/run.py @@ -9,13 +9,13 @@ class TestRunner: - def __init__(self, local_net=True): + def __init__(self, local_net=True,config_file=None, argv=None): LOGGER.info('Starting Test Run') - testrun = TestRun(local_net) + testrun = TestRun(local_net=local_net,argsv=argv) - testrun.load_config() + testrun.load_config(config_file=config_file) testrun.start_network() @@ -30,10 +30,12 @@ def run(argv): parser.add_argument("-r", "--remote-net", action="store_false", help='''Use the network orchestrator from the parent directory instead of the one downloaded locally from the install script.''') + parser.add_argument("-f","--config-file", default=None, + help="Define the configuration file for Test Run and Network Orchestrator") args, unknown = parser.parse_known_args() - TestRunner(args.remote_net) + TestRunner(local_net=args.remote_net,config_file=args.config_file, argv=argv) if __name__ == "__main__": diff --git a/framework/testrun.py b/framework/testrun.py index 22fa0295a..e4da47cc5 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -6,7 +6,6 @@ Run using the provided command scripts in the cmd folder. E.g sudo cmd/start """ - import os import sys import json @@ -22,25 +21,27 @@ EXAMPLE_CONFIG_FILE = "conf/system.json.example" RUNTIME = 300 -class TestRun: # pylint: disable=too-few-public-methods + +class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. Creates an instance of the network orchestrator, test orchestrator and user interface. """ - def __init__(self,local_net=True): + def __init__(self, local_net=True, argsv=None): # Catch any exit signals self._register_exits() # Import the correct net orchestrator - self.import_orchestrators(local_net) + self.import_dependencies(local_net) self._net_orc = net_orc.NetworkOrchestrator() self._test_orc = test_orc.TestOrchestrator() + self._net_run = net_run.NetworkRunner(argsv) - def import_orchestrators(self,local_net=True): + def import_dependencies(self, local_net=True): if local_net: # Add local net_orc to Python path net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') @@ -48,18 +49,22 @@ def import_orchestrators(self,local_net=True): # Resolve the path to the test-run parent folder root_dir = os.path.abspath(os.path.join(parent_dir, os.pardir)) # Add manually cloned network orchestrator from parent folder - net_orc_dir = os.path.join(root_dir, 'network-orchestrator', 'python', 'src') + net_orc_dir = os.path.join( + root_dir, 'network-orchestrator', 'python', 'src') # Add net_orc to Python path sys.path.append(net_orc_dir) # Import the network orchestrator global net_orc - import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel + import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel # Add test_orc to Python path test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') sys.path.append(test_orc_dir) global test_orc - import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel + import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel + + global net_run + import network_runner as net_run # pylint: disable=wrong-import-position,import-outside-toplevel def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -67,41 +72,34 @@ def _register_exits(self): signal.signal(signal.SIGABRT, self._exit_handler) signal.signal(signal.SIGQUIT, self._exit_handler) - def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument LOGGER.debug("Exit signal received: " + str(signum)) if signum in (2, signal.SIGTERM): LOGGER.info("Exit signal received.") self.stop_network() - def load_config(self): + def load_config(self, config_file=None): + if config_file is None: + # If not defined, use relative pathing to local file + self._config_file = os.path.join(parent_dir, CONFIG_FILE) + else: + # If defined, use as provided + self._config_file = config_file + """Loads all settings from the config file into memory.""" - if not os.path.isfile(CONFIG_FILE): - LOGGER.error("Configuration file is not present at " + CONFIG_FILE) + if not os.path.isfile(self._config_file): + LOGGER.error("Configuration file is not present at " + self._config_file) LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) sys.exit(1) - with open(CONFIG_FILE, 'r', encoding='UTF-8') as config_file_open: - config_json = json.load(config_file_open) - self._net_orc.import_config(config_json) + LOGGER.info("Loading Config File: " + os.path.abspath(self._config_file)) + + with open(self._config_file, encoding='UTF-8') as config_json_file: + config_json = json.load(config_json_file) self._test_orc.import_config(config_json) def start_network(self): - """Starts the network orchestrator and network services.""" - - # Load and build any unbuilt network containers - self._net_orc.load_network_modules() - self._net_orc.build_network_modules() - - self._net_orc.stop_networking_services(kill=True) - self._net_orc.restore_net() - - # Create baseline network - self._net_orc.create_net() - - # Launch network service containers - self._net_orc.start_network_services() - - LOGGER.info("Network is ready.") + self._net_run.run() def run_tests(self): """Iterate through and start all test modules.""" @@ -113,7 +111,4 @@ def run_tests(self): self._test_orc.run_test_modules() def stop_network(self): - """Commands the net_orc to stop the network and clean up.""" - self._net_orc.stop_networking_services(kill=True) - self._net_orc.restore_net() - sys.exit(0) + self._net_run._stop() From c5e56bef1bfc97c92215aaea432b3e1e27c6342f Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 26 Apr 2023 12:25:12 -0600 Subject: [PATCH 17/20] fix merge issues --- framework/run.py | 4 ++-- framework/testrun.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/framework/run.py b/framework/run.py index db53bd680..81106d174 100644 --- a/framework/run.py +++ b/framework/run.py @@ -9,11 +9,11 @@ class TestRunner: - def __init__(self, local_net=True): + def __init__(self, local_net=True,config_file=None, argv=None): LOGGER.info('Starting Test Run') - testrun = TestRun(local_net) + testrun = TestRun(local_net=local_net,argsv=argv) testrun.load_config() diff --git a/framework/testrun.py b/framework/testrun.py index fd6cce0a6..cb1436868 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -38,6 +38,7 @@ class TestRun: # pylint: disable=too-few-public-methods """ def __init__(self, local_net=True, argsv=None): + self._devices = [] # Catch any exit signals self._register_exits() @@ -56,9 +57,10 @@ def start(self): self.start_network() # Register callbacks - self._net_orc.listener.register_callback( - self._device_discovered, - [NetworkEvent.DEVICE_DISCOVERED]) + # Disable for now as this is causing boot failures when no devices are discovered + # self._net_orc.listener.register_callback( + # self._device_discovered, + # [NetworkEvent.DEVICE_DISCOVERED]) def import_dependencies(self, local_net=True): if local_net: @@ -85,6 +87,9 @@ def import_dependencies(self, local_net=True): global net_run import network_runner as net_run # pylint: disable=wrong-import-position,import-outside-toplevel + global NetworkEvent + from listener import NetworkEvent # pylint: disable=wrong-import-position,import-outside-toplevel + def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) signal.signal(signal.SIGTERM, self._exit_handler) @@ -120,7 +125,7 @@ def load_config(self, config_file=None): self._test_orc.import_config(config_json) def start_network(self): - self._net_run.run() + self._net_run.start(async_monitor=True) def run_tests(self): """Iterate through and start all test modules.""" @@ -167,3 +172,4 @@ def _device_discovered(self, mac_addr): else: LOGGER.info( f'A new device has been discovered with mac address {device.mac_addr}') + return device From 424205fa510cac7fbb2e058d77cfb3d57c0600a1 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 27 Apr 2023 12:35:11 -0600 Subject: [PATCH 18/20] Update runner and test orch procedure Add useful runtiem args --- .gitignore | 270 +++++++++++++++++++-------------------- cmd/install | 2 +- cmd/start | 2 +- framework/run.py | 41 ------ framework/test_runner.py | 73 +++++++++++ framework/testrun.py | 29 +++-- 6 files changed, 230 insertions(+), 187 deletions(-) delete mode 100644 framework/run.py create mode 100644 framework/test_runner.py diff --git a/.gitignore b/.gitignore index 4016b6901..f79a6efcb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,135 +1,135 @@ -# Runtime folder -runtime/ -venv/ -net_orc/ -.vscode/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +# Runtime folder +runtime/ +venv/ +net_orc/ +.vscode/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cmd/install b/cmd/install index 61722e273..42c26c75e 100755 --- a/cmd/install +++ b/cmd/install @@ -2,7 +2,7 @@ GIT_URL=https://github.com/auto-iot NET_ORC_DIR=net_orc -NET_ORC_VERSION="dev" +NET_ORC_VERSION="test_run_sync" python3 -m venv venv diff --git a/cmd/start b/cmd/start index fa6bbc1e1..113f14b3e 100755 --- a/cmd/start +++ b/cmd/start @@ -18,6 +18,6 @@ rm -rf runtime source venv/bin/activate # TODO: Execute python code -python -u framework/run.py $@ +python -u framework/test_runner.py $@ deactivate \ No newline at end of file diff --git a/framework/run.py b/framework/run.py deleted file mode 100644 index 81106d174..000000000 --- a/framework/run.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Starts Test Run.""" - -import argparse -import sys -from testrun import TestRun -import logger - -LOGGER = logger.get_logger('runner') - -class TestRunner: - - def __init__(self, local_net=True,config_file=None, argv=None): - - LOGGER.info('Starting Test Run') - - testrun = TestRun(local_net=local_net,argsv=argv) - - testrun.load_config() - - testrun.start() - - testrun.run_tests() - - testrun.stop_network() - - -def run(argv): - parser = argparse.ArgumentParser(description="Test Run", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-r", "--remote-net", action="store_false", - help='''Use the network orchestrator from the parent directory instead - of the one downloaded locally from the install script.''') - parser.add_argument("-f","--config-file", default=None, - help="Define the configuration file for Test Run and Network Orchestrator") - - args, unknown = parser.parse_known_args() - - TestRunner(local_net=args.remote_net,config_file=args.config_file, argv=argv) - -if __name__ == "__main__": - run(sys.argv) diff --git a/framework/test_runner.py b/framework/test_runner.py new file mode 100644 index 000000000..f259c43d6 --- /dev/null +++ b/framework/test_runner.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +"""Wrapper for the TestRun that simplifies +virtual testing procedure by allowing direct calling +from the command line. + +Run using the provided command scripts in the cmd folder. +E.g sudo cmd/start +""" + +import argparse +import sys +from testrun import TestRun +import logger +import signal + +LOGGER = logger.get_logger('runner') + + +class TestRunner: + + def __init__(self, local_net=True, config_file=None, validate=True, net_only=False): + self._register_exits() + self.test_run = TestRun(local_net=local_net, config_file=config_file, + validate=validate, net_only=net_only) + + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) + + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received.") + # Kill all container services quickly + # If we're here, we want everything to stop immediately + # and don't care about a gracefully shutdown + self._stop(True) + sys.exit(1) + + def stop(self, kill=False): + self.test_run.stop(kill) + + def start(self): + self.test_run.start() + LOGGER.info("TestRunner Done") + + +def parse_args(argv): + parser = argparse.ArgumentParser(description="Test Run", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-r", "--remote-net", action="store_false", + help='''Use the network orchestrator from the parent directory instead + of the one downloaded locally from the install script.''') + parser.add_argument("-f", "--config-file", default=None, + help="Define the configuration file for Test Run and Network Orchestrator") + parser.add_argument("--no-validate", action="store_true", + help="Turn off the validation of the network after network boot") + parser.add_argument("-net", "--net-only", action="store_true", + help="Run the network only, do not run tests") + args, unknown = parser.parse_known_args() + return args + + +if __name__ == "__main__": + args = parse_args(sys.argv) + runner = TestRunner(local_net=args.remote_net, + config_file=args.config_file, + validate=not args.no_validate, + net_only=args.net_only) + runner.start() diff --git a/framework/testrun.py b/framework/testrun.py index cb1436868..d306684e2 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -37,8 +37,9 @@ class TestRun: # pylint: disable=too-few-public-methods orchestrator and user interface. """ - def __init__(self, local_net=True, argsv=None): + def __init__(self, local_net=True, config_file=CONFIG_FILE,validate=True, net_only=False): self._devices = [] + self._net_only = net_only # Catch any exit signals self._register_exits() @@ -46,15 +47,22 @@ def __init__(self, local_net=True, argsv=None): # Import the correct net orchestrator self.import_dependencies(local_net) - self._net_orc = net_orc.NetworkOrchestrator() + #self._net_orc = net_orc.NetworkOrchestrator() self._test_orc = test_orc.TestOrchestrator() - self._net_run = net_run.NetworkRunner(argsv) + self._net_run = net_run.NetworkRunner(config_file=config_file,validate=validate,async_monitor=not self._net_only) def start(self): self._load_devices() - self.start_network() + if self._net_only: + LOGGER.info("Network only configured, no tests will be run") + self._start_network() + else: + self._start_network() + self._run_tests() + + self.stop() # Register callbacks # Disable for now as this is causing boot failures when no devices are discovered @@ -62,6 +70,9 @@ def start(self): # self._device_discovered, # [NetworkEvent.DEVICE_DISCOVERED]) + def stop(self): + self._stop_network() + def import_dependencies(self, local_net=True): if local_net: # Add local net_orc to Python path @@ -124,10 +135,10 @@ def load_config(self, config_file=None): config_json = json.load(config_json_file) self._test_orc.import_config(config_json) - def start_network(self): - self._net_run.start(async_monitor=True) + def _start_network(self): + self._net_run.start() - def run_tests(self): + def _run_tests(self): """Iterate through and start all test modules.""" self._test_orc.load_test_modules() @@ -136,8 +147,8 @@ def run_tests(self): # Begin testing self._test_orc.run_test_modules() - def stop_network(self): - self._net_run._stop() + def _stop_network(self): + self._net_run.stop() def _load_devices(self): LOGGER.debug('Loading devices from ' + DEVICES_DIR) From f57e8caf697f67c7a72150501de9b5f32fafc2f1 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 27 Apr 2023 16:46:42 -0600 Subject: [PATCH 19/20] Restructure test run startup process Misc updates to work with net orch updates --- framework/testrun.py | 60 +++++++++--------------- test_orc/python/src/test_orchestrator.py | 28 +++++------ 2 files changed, 36 insertions(+), 52 deletions(-) diff --git a/framework/testrun.py b/framework/testrun.py index d306684e2..4a29b4e20 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -47,20 +47,22 @@ def __init__(self, local_net=True, config_file=CONFIG_FILE,validate=True, net_on # Import the correct net orchestrator self.import_dependencies(local_net) - #self._net_orc = net_orc.NetworkOrchestrator() + # Expand the config file to absolute pathing + config_file_abs=self._get_config_abs(config_file=config_file) + + self._net_orc = net_orc.NetworkOrchestrator(config_file=config_file_abs,validate=validate,async_monitor=not self._net_only) self._test_orc = test_orc.TestOrchestrator() - self._net_run = net_run.NetworkRunner(config_file=config_file,validate=validate,async_monitor=not self._net_only) - + def start(self): self._load_devices() if self._net_only: - LOGGER.info("Network only configured, no tests will be run") + LOGGER.info("Network only option configured, no tests will be run") self._start_network() else: self._start_network() - self._run_tests() + self._start_tests() self.stop() @@ -70,8 +72,9 @@ def start(self): # self._device_discovered, # [NetworkEvent.DEVICE_DISCOVERED]) - def stop(self): - self._stop_network() + def stop(self,kill=False): + self._stop_tests() + self._stop_network(kill=kill) def import_dependencies(self, local_net=True): if local_net: @@ -95,9 +98,6 @@ def import_dependencies(self, local_net=True): global test_orc import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel - global net_run - import network_runner as net_run # pylint: disable=wrong-import-position,import-outside-toplevel - global NetworkEvent from listener import NetworkEvent # pylint: disable=wrong-import-position,import-outside-toplevel @@ -111,44 +111,30 @@ def _exit_handler(self, signum, arg): # pylint: disable=unused-argument LOGGER.debug("Exit signal received: " + str(signum)) if signum in (2, signal.SIGTERM): LOGGER.info("Exit signal received.") - self.stop_network() + self.stop(kill=True) + sys.exit(1) - def load_config(self, config_file=None): + def _get_config_abs(self,config_file=None): if config_file is None: # If not defined, use relative pathing to local file - self._config_file = os.path.join(parent_dir, CONFIG_FILE) - else: - # If defined, use as provided - self._config_file = config_file - - """Loads all settings from the config file into memory.""" - if not os.path.isfile(self._config_file): - LOGGER.error( - "Configuration file is not present at " + self._config_file) - LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) - sys.exit(1) - - LOGGER.info("Loading Config File: " + - os.path.abspath(self._config_file)) + config_file = os.path.join(parent_dir, CONFIG_FILE) - with open(self._config_file, encoding='UTF-8') as config_json_file: - config_json = json.load(config_json_file) - self._test_orc.import_config(config_json) + # Expand the config file to absolute pathing + return os.path.abspath(config_file) def _start_network(self): - self._net_run.start() + self._net_orc.start() - def _run_tests(self): + def _start_tests(self): """Iterate through and start all test modules.""" - self._test_orc.load_test_modules() - self._test_orc.build_test_modules() + self._test_orc.start() - # Begin testing - self._test_orc.run_test_modules() + def _stop_network(self,kill=False): + self._net_orc.stop(kill=kill) - def _stop_network(self): - self._net_run.stop() + def _stop_tests(self): + self._test_orc.stop() def _load_devices(self): LOGGER.debug('Loading devices from ' + DEVICES_DIR) diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 396f533fa..ccf59e0f3 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -29,26 +29,23 @@ def __init__(self): shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR), ignore_errors=True) os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) - def import_config(self, json_config): - """Load settings from JSON object into memory.""" + def start(self): + LOGGER.info("Starting Test Orchestrator") + self._load_test_modules() + self._run_test_modules() - # No relevant config options in system.json as of yet + def stop(self): + """Stop any running tests""" + self._stop_modules() - def get_test_module(self, name): - """Returns a test module by the module name.""" - for module in self._test_modules: - if name == module.name: - return module - return None - - def run_test_modules(self): + def _run_test_modules(self): """Iterates through each test module and starts the container.""" LOGGER.info("Running test modules...") for module in self._test_modules: - self.run_test_module(module) + self._run_test_module(module) LOGGER.info("All tests complete") - def run_test_module(self, module): + def _run_test_module(self, module): """Start the test container and extract the results.""" if module is None or not module.enable_container: @@ -111,7 +108,7 @@ def _get_module_container(self, module): LOGGER.error(error) return container - def load_test_modules(self): + def _load_test_modules(self): """Import module configuration from module_config.json.""" modules_dir = os.path.join(self._path, TEST_MODULES_DIR) @@ -175,12 +172,13 @@ def _build_test_module(self, module): LOGGER.error(error) def _stop_modules(self, kill=False): - LOGGER.debug("Stopping test modules") + LOGGER.info("Stopping test modules") for module in self._test_modules: # Test modules may just be Docker images, so we do not want to stop them if not module.enable_container: continue self._stop_module(module, kill) + LOGGER.info("Test Modules Stopped") def _stop_module(self, module, kill=False): LOGGER.debug("Stopping test module " + module.container_name) From 035bd71d57401bad77d934f0fe9a8c9f4275185f Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Fri, 28 Apr 2023 13:39:49 +0100 Subject: [PATCH 20/20] Refactor --- cmd/install | 2 +- framework/test_runner.py | 2 +- test_orc/python/src/module.py | 23 ++++++++++++++ test_orc/python/src/runner.py | 40 ++++++++++++++++++++++++ test_orc/python/src/test_orchestrator.py | 16 +++++----- 5 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 test_orc/python/src/module.py create mode 100644 test_orc/python/src/runner.py diff --git a/cmd/install b/cmd/install index 42c26c75e..6dee1c635 100755 --- a/cmd/install +++ b/cmd/install @@ -2,7 +2,7 @@ GIT_URL=https://github.com/auto-iot NET_ORC_DIR=net_orc -NET_ORC_VERSION="test_run_sync" +NET_ORC_VERSION="main" python3 -m venv venv diff --git a/framework/test_runner.py b/framework/test_runner.py index f259c43d6..91ff4cb1a 100644 --- a/framework/test_runner.py +++ b/framework/test_runner.py @@ -45,7 +45,7 @@ def stop(self, kill=False): def start(self): self.test_run.start() - LOGGER.info("TestRunner Done") + LOGGER.info("Test Run has finished") def parse_args(argv): diff --git a/test_orc/python/src/module.py b/test_orc/python/src/module.py new file mode 100644 index 000000000..6d24d7e1e --- /dev/null +++ b/test_orc/python/src/module.py @@ -0,0 +1,23 @@ +"""Represemts a test module.""" +from dataclasses import dataclass +from docker.client.Container import Container + +@dataclass +class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes + """Represents a test module.""" + + name: str = None + display_name: str = None + description: str = None + + build_file: str = None + container: Container = None + container_name: str = None + image_name :str = None + enable_container: bool = True + + timeout: int = 60 + + # Absolute path + dir: str = None + dir_name: str = None diff --git a/test_orc/python/src/runner.py b/test_orc/python/src/runner.py new file mode 100644 index 000000000..cc495bf8d --- /dev/null +++ b/test_orc/python/src/runner.py @@ -0,0 +1,40 @@ +"""Provides high level management of the test orchestrator.""" +import time +import logger + +LOGGER = logger.get_logger('runner') + +class Runner: + """Holds the state of the testing for one device.""" + + def __init__(self, test_orc, device): + self._test_orc = test_orc + self._device = device + + def run(self): + self._run_test_modules() + + def _run_test_modules(self): + """Iterates through each test module and starts the container.""" + LOGGER.info('Running test modules...') + for module in self._test_modules: + self.run_test_module(module) + LOGGER.info('All tests complete') + + def run_test_module(self, module): + """Start the test container and extract the results.""" + + if module is None or not module.enable_container: + return + + self._test_orc.start_test_module(module) + + # Determine the module timeout time + test_module_timeout = time.time() + module.timeout + status = self._test_orc.get_module_status(module) + + while time.time() < test_module_timeout and status == 'running': + time.sleep(1) + status = self._test_orc.get_module_status(module) + + LOGGER.info(f'Test module {module.display_name} has finished') diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index ccf59e0f3..77f73f407 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -51,7 +51,7 @@ def _run_test_module(self, module): if module is None or not module.enable_container: return - LOGGER.info("Running test module " + module.display_name) + LOGGER.info("Running test module " + module.name) try: container_runtime_dir = os.path.join(self._root_path, "runtime/test/" + module.name) @@ -75,7 +75,7 @@ def _run_test_module(self, module): environment={"HOST_USER": os.getlogin()} ) except (docker.errors.APIError, docker.errors.ContainerError) as container_error: - LOGGER.error("Test module " + module.display_name + " has failed to start") + LOGGER.error("Test module " + module.name + " has failed to start") LOGGER.debug(container_error) return @@ -87,7 +87,7 @@ def _run_test_module(self, module): time.sleep(1) status = self._get_module_status(module) - LOGGER.info("Test module " + module.display_name + " has finished") + LOGGER.info("Test module " + module.name + " has finished") def _get_module_status(self,module): container = self._get_module_container(module) @@ -148,7 +148,8 @@ def _load_test_modules(self): self._test_modules.append(module) - loaded_modules += module.dir_name + " " + if module.enable_container: + loaded_modules += module.dir_name + " " LOGGER.info(loaded_modules) @@ -178,7 +179,7 @@ def _stop_modules(self, kill=False): if not module.enable_container: continue self._stop_module(module, kill) - LOGGER.info("Test Modules Stopped") + LOGGER.info("All test modules have been stopped") def _stop_module(self, module, kill=False): LOGGER.debug("Stopping test module " + module.container_name) @@ -194,9 +195,8 @@ def _stop_module(self, module, kill=False): module.container_name) container.stop() LOGGER.debug("Container stopped:" + module.container_name) - except Exception as error: - LOGGER.error("Container stop error") - LOGGER.error(error) + except docker.errors.NotFound: + pass class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Represents a test module."""