From 125f08fd16f9beb98461b86488e8a2bbcba9baab Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 4 Jul 2023 13:13:31 +0100 Subject: [PATCH 01/32] Start API --- framework/python/src/api/api.py | 117 ++++++++++++++++++ framework/python/src/api/system_interfaces.py | 5 + .../python/src/{core => common}/device.py | 0 framework/python/src/core/testrun.py | 64 +++------- .../src/net_orc/network_orchestrator.py | 15 ++- framework/requirements.txt | 8 +- 6 files changed, 159 insertions(+), 50 deletions(-) create mode 100644 framework/python/src/api/api.py create mode 100644 framework/python/src/api/system_interfaces.py rename framework/python/src/{core => common}/device.py (100%) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py new file mode 100644 index 000000000..65979fd71 --- /dev/null +++ b/framework/python/src/api/api.py @@ -0,0 +1,117 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fastapi import FastAPI, APIRouter +import json +import logging +import os +import psutil +import threading +import uvicorn + +from api.system_interfaces import SystemInterfaces +from common import logger, util +from common.device import Device + +LOGGER = logger.get_logger("api") +LOCAL_DEVICES_DIR = "local/devices" +RESOURCE_DEVICES_DIR = "resources/devices" +DEVICE_CONFIG = "device_config.json" +DEVICE_MANUFACTURER = "manufacturer" +DEVICE_MODEL = "model" +DEVICE_MAC_ADDR = "mac_addr" +DEVICE_TEST_MODULES = "test_modules" + +class Api: + """Provide REST endpoints to manage Test Run""" + + def __init__(self, test_run): + + self._test_run = test_run + self._name = "TestRun API" + self._router = APIRouter() + + self._devices = [] + self._config_file_url = self._test_run.get_config_file() + + self._router.add_api_route("/system/interfaces", self.sys_interfaces) + self._router.add_api_route("/system/interfaces", self.post_sys_interfaces, + methods=["POST"]) + self._router.add_api_route("/devices", self.get_devices) + + self._app = FastAPI() + self._app.include_router(self._router) + + self._api_thread = threading.Thread(target=self._start, + name="Test Run API", + daemon=True) + + def start(self): + LOGGER.info("Starting API") + self._api_thread.start() + + def _start(self): + + uvicorn_error = logging.getLogger("uvicorn.error") + uvicorn_error.propagate = False + uvicorn_access = logging.getLogger("uvicorn.access") + uvicorn_access.propagate = False + + uvicorn.run(self._app) + + def stop(self): + LOGGER.info("Stopping API") + + def load_all_devices(self): + self._load_devices(device_dir=LOCAL_DEVICES_DIR) + self._load_devices(device_dir=RESOURCE_DEVICES_DIR) + return self._devices + + def _load_devices(self, device_dir): + LOGGER.debug("Loading devices from " + device_dir) + + os.makedirs(device_dir, exist_ok=True) + util.run_command(f"chown -R {util.get_host_user()} {device_dir}") + + for device_folder in os.listdir(device_dir): + with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), + encoding="utf-8") as device_config_file: + device_config_json = json.load(device_config_file) + + device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) + device_model = device_config_json.get(DEVICE_MODEL) + mac_addr = device_config_json.get(DEVICE_MAC_ADDR) + test_modules = device_config_json.get(DEVICE_TEST_MODULES) + + device = Device(manufacturer=device_manufacturer, + model=device_model, + mac_addr=mac_addr, + test_modules=json.dumps(test_modules)) + self._devices.append(device) + + async def sys_interfaces(self): + addrs = psutil.net_if_addrs() + ifaces = [] + for iface in addrs: + ifaces.append(iface) + return ifaces + + async def post_sys_interfaces(self, sys_intfs: SystemInterfaces): + + # TODO: Write config file + print(sys_intfs) + return sys_intfs + + async def get_devices(self): + return self._devices diff --git a/framework/python/src/api/system_interfaces.py b/framework/python/src/api/system_interfaces.py new file mode 100644 index 000000000..3abee8b7f --- /dev/null +++ b/framework/python/src/api/system_interfaces.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + +class SystemInterfaces(BaseModel): + device_intf: str + internet_intf: str diff --git a/framework/python/src/core/device.py b/framework/python/src/common/device.py similarity index 100% rename from framework/python/src/core/device.py rename to framework/python/src/common/device.py diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index a91736e95..f05a7997a 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -22,10 +22,10 @@ """ import os import sys -import json import signal import time -from common import logger, util +from common import logger +from common.device import Device # Locate parent directory current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -33,24 +33,16 @@ # Locate the test-run root directory, 4 levels, src->python->framework->test-run root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) +from api.api import Api from net_orc.listener import NetworkEvent from test_orc import test_orchestrator as test_orc from net_orc import network_orchestrator as net_orc -from device import Device LOGGER = logger.get_logger('test_run') -CONFIG_FILE = 'local/system.json' +DEFAULT_CONFIG_FILE = 'local/system.json' EXAMPLE_CONFIG_FILE = 'local/system.json.example' RUNTIME = 120 -LOCAL_DEVICES_DIR = 'local/devices' -RESOURCE_DEVICES_DIR = 'resources/devices' -DEVICE_CONFIG = 'device_config.json' -DEVICE_MANUFACTURER = 'manufacturer' -DEVICE_MODEL = 'model' -DEVICE_MAC_ADDR = 'mac_addr' -DEVICE_TEST_MODULES = 'test_modules' - class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. @@ -59,19 +51,21 @@ class TestRun: # pylint: disable=too-few-public-methods """ def __init__(self, - config_file=CONFIG_FILE, + config_file=DEFAULT_CONFIG_FILE, validate=True, net_only=False, single_intf=False): + self._devices = [] self._net_only = net_only self._single_intf = single_intf + self._config_file = config_file # Catch any exit signals self._register_exits() # Expand the config file to absolute pathing - config_file_abs = self._get_config_abs(config_file=config_file) + config_file_abs = self._get_config_abs(config_file=self._config_file) self._net_orc = net_orc.NetworkOrchestrator( config_file=config_file_abs, @@ -80,9 +74,11 @@ def __init__(self, self._test_orc = test_orc.TestOrchestrator(self._net_orc) - def start(self): + self._api = Api(self) + self._devices = self._api.load_all_devices() + self._api.start() - self._load_all_devices() + def start(self): self._start_network() @@ -129,6 +125,7 @@ def start(self): def stop(self, kill=False): self._stop_tests() self._stop_network(kill=kill) + self._api.stop() def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -146,14 +143,19 @@ def _exit_handler(self, signum, arg): # pylint: disable=unused-argument def _get_config_abs(self, config_file=None): if config_file is None: # If not defined, use relative pathing to local file - config_file = os.path.join(root_dir, CONFIG_FILE) + config_file = os.path.join(root_dir, self._config_file) # Expand the config file to absolute pathing return os.path.abspath(config_file) + def get_config_file(self): + return self._get_config_abs() + def _start_network(self): # Start the network orchestrator - self._net_orc.start() + if not self._net_orc.start(): + self.stop(kill=True) + sys.exit(1) def _stop_network(self, kill=False): self._net_orc.stop(kill=kill) @@ -161,32 +163,6 @@ def _stop_network(self, kill=False): def _stop_tests(self): self._test_orc.stop() - def _load_all_devices(self): - self._load_devices(device_dir=LOCAL_DEVICES_DIR) - self._load_devices(device_dir=RESOURCE_DEVICES_DIR) - - def _load_devices(self, device_dir): - LOGGER.debug('Loading devices from ' + device_dir) - - os.makedirs(device_dir, exist_ok=True) - util.run_command(f'chown -R {util.get_host_user()} {device_dir}') - - for device_folder in os.listdir(device_dir): - with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), - encoding='utf-8') as device_config_file: - device_config_json = json.load(device_config_file) - - device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) - device_model = device_config_json.get(DEVICE_MODEL) - mac_addr = device_config_json.get(DEVICE_MAC_ADDR) - test_modules = device_config_json.get(DEVICE_TEST_MODULES) - - device = Device(manufacturer=device_manufacturer, - model=device_model, - mac_addr=mac_addr, - test_modules=json.dumps(test_modules)) - self._devices.append(device) - def get_device(self, mac_addr): """Returns a loaded device object from the device mac address.""" for device in self._devices: diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 499ce954b..885d5f75a 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + """Network orchestrator is responsible for managing all of the virtual network services""" import getpass @@ -23,8 +24,7 @@ import sys import docker from docker.types import Mount -from common import logger -from common import util +from common import logger, util from net_orc.listener import Listener from net_orc.network_device import NetworkDevice from net_orc.network_event import NetworkEvent @@ -100,14 +100,15 @@ def start(self): # Restore the network first if required self.stop(kill=True) - self.start_network() + return self.start_network() def start_network(self): """Start the virtual testing network.""" LOGGER.info('Starting network') self.build_network_modules() - self.create_net() + if not self.create_net(): + return False self.start_network_services() if self.validate: @@ -117,6 +118,8 @@ def start_network(self): # Get network ready (via Network orchestrator) LOGGER.debug('Network is ready') + return True + def start_listener(self): self.listener.start_listener() @@ -320,7 +323,7 @@ def create_net(self): self._dev_intf): LOGGER.error('Configured interfaces are not ready for use. ' + 'Ensure both interfaces are connected.') - sys.exit(1) + return False if self._single_intf: self._ci_pre_network_create() @@ -345,6 +348,8 @@ def create_net(self): self.listener.register_callback(self._dhcp_lease_ack, [NetworkEvent.DHCP_LEASE_ACK]) + return True + def load_network_modules(self): """Load network modules from module_config.json.""" LOGGER.debug('Loading network modules from /' + NETWORK_MODULES_DIR) diff --git a/framework/requirements.txt b/framework/requirements.txt index 03eab9796..5a47cd056 100644 --- a/framework/requirements.txt +++ b/framework/requirements.txt @@ -5,4 +5,10 @@ requests<2.29.0 docker ipaddress netifaces -scapy \ No newline at end of file +scapy + +# Requirements for the API +fastapi +psutil +uvicorn +pydantic \ No newline at end of file From f47c1d0e30998d5d170f904b9196a46e08cf5336 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Fri, 7 Jul 2023 09:47:30 +0100 Subject: [PATCH 02/32] Write interfaces --- framework/python/src/api/api.py | 24 +++++++++++++----------- framework/python/src/core/testrun.py | 5 ++++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 65979fd71..21b26bc60 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -46,7 +46,7 @@ def __init__(self, test_run): self._config_file_url = self._test_run.get_config_file() self._router.add_api_route("/system/interfaces", self.sys_interfaces) - self._router.add_api_route("/system/interfaces", self.post_sys_interfaces, + self._router.add_api_route("/system/interfaces", self.post_sys_interfaces, methods=["POST"]) self._router.add_api_route("/devices", self.get_devices) @@ -62,13 +62,7 @@ def start(self): self._api_thread.start() def _start(self): - - uvicorn_error = logging.getLogger("uvicorn.error") - uvicorn_error.propagate = False - uvicorn_access = logging.getLogger("uvicorn.access") - uvicorn_access.propagate = False - - uvicorn.run(self._app) + uvicorn.run(self._app, log_config=None) def stop(self): LOGGER.info("Stopping API") @@ -97,7 +91,7 @@ def _load_devices(self, device_dir): device = Device(manufacturer=device_manufacturer, model=device_model, mac_addr=mac_addr, - test_modules=json.dumps(test_modules)) + test_modules=test_modules) self._devices.append(device) async def sys_interfaces(self): @@ -109,8 +103,16 @@ async def sys_interfaces(self): async def post_sys_interfaces(self, sys_intfs: SystemInterfaces): - # TODO: Write config file - print(sys_intfs) + config_file = open(self._config_file_url, "r", encoding="utf-8") + json_contents = json.load(config_file) + config_file.close() + + json_contents["network"]["device_intf"] = sys_intfs.device_intf + json_contents["network"]["internet_intf"] = sys_intfs.internet_intf + + with open(self._config_file_url, "w", encoding="utf-8") as config_file: + json.dump(json_contents, config_file, indent=2) + return sys_intfs async def get_devices(self): diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index f05a7997a..039c81951 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -51,11 +51,14 @@ class TestRun: # pylint: disable=too-few-public-methods """ def __init__(self, - config_file=DEFAULT_CONFIG_FILE, + config_file, validate=True, net_only=False, single_intf=False): + if config_file is None: + config_file = DEFAULT_CONFIG_FILE + self._devices = [] self._net_only = net_only self._single_intf = single_intf From ceaac38bfe82b5b6d3798d829e67dec1e3ad9db4 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Fri, 7 Jul 2023 11:55:25 +0100 Subject: [PATCH 03/32] Get current configuration --- framework/python/src/api/api.py | 24 ++++++++++++------- framework/python/src/api/system_config.py | 5 ++++ ...interfaces.py => system_network_config.py} | 2 +- framework/python/src/core/testrun.py | 2 ++ 4 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 framework/python/src/api/system_config.py rename framework/python/src/api/{system_interfaces.py => system_network_config.py} (65%) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 21b26bc60..8728d36a1 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -14,13 +14,12 @@ from fastapi import FastAPI, APIRouter import json -import logging import os import psutil import threading import uvicorn -from api.system_interfaces import SystemInterfaces +from api.system_config import SystemConfig from common import logger, util from common.device import Device @@ -45,9 +44,10 @@ def __init__(self, test_run): self._devices = [] self._config_file_url = self._test_run.get_config_file() - self._router.add_api_route("/system/interfaces", self.sys_interfaces) - self._router.add_api_route("/system/interfaces", self.post_sys_interfaces, + self._router.add_api_route("/system/interfaces", self.get_sys_interfaces) + self._router.add_api_route("/system/config", self.post_sys_config, methods=["POST"]) + self._router.add_api_route("/system/config", self.get_sys_config) self._router.add_api_route("/devices", self.get_devices) self._app = FastAPI() @@ -94,26 +94,32 @@ def _load_devices(self, device_dir): test_modules=test_modules) self._devices.append(device) - async def sys_interfaces(self): + async def get_sys_interfaces(self): addrs = psutil.net_if_addrs() ifaces = [] for iface in addrs: ifaces.append(iface) return ifaces - async def post_sys_interfaces(self, sys_intfs: SystemInterfaces): + async def post_sys_config(self, sys_config: SystemConfig): config_file = open(self._config_file_url, "r", encoding="utf-8") json_contents = json.load(config_file) config_file.close() - json_contents["network"]["device_intf"] = sys_intfs.device_intf - json_contents["network"]["internet_intf"] = sys_intfs.internet_intf + json_contents["network"]["device_intf"] = sys_config.network.device_intf + json_contents["network"]["internet_intf"] = sys_config.network.internet_intf with open(self._config_file_url, "w", encoding="utf-8") as config_file: json.dump(json_contents, config_file, indent=2) - return sys_intfs + return sys_config + + async def get_sys_config(self): + config_file = open(self._config_file_url, "r", encoding="utf-8") + json_contents = json.load(config_file) + config_file.close() + return json_contents async def get_devices(self): return self._devices diff --git a/framework/python/src/api/system_config.py b/framework/python/src/api/system_config.py new file mode 100644 index 000000000..75ee2fb9d --- /dev/null +++ b/framework/python/src/api/system_config.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel +from api.system_network_config import SystemNetworkConfig + +class SystemConfig(BaseModel): + network: SystemNetworkConfig diff --git a/framework/python/src/api/system_interfaces.py b/framework/python/src/api/system_network_config.py similarity index 65% rename from framework/python/src/api/system_interfaces.py rename to framework/python/src/api/system_network_config.py index 3abee8b7f..78a511491 100644 --- a/framework/python/src/api/system_interfaces.py +++ b/framework/python/src/api/system_network_config.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -class SystemInterfaces(BaseModel): +class SystemNetworkConfig(BaseModel): device_intf: str internet_intf: str diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 039c81951..30f947fab 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -81,6 +81,8 @@ def __init__(self, self._devices = self._api.load_all_devices() self._api.start() + time.sleep(100) + def start(self): self._start_network() From 6e5cc2e3b906a759fc89a3cad673e5ac84bc07a3 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Fri, 7 Jul 2023 12:20:19 +0100 Subject: [PATCH 04/32] Set versions --- framework/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/requirements.txt b/framework/requirements.txt index 5a47cd056..560c2baf9 100644 --- a/framework/requirements.txt +++ b/framework/requirements.txt @@ -8,7 +8,7 @@ netifaces scapy # Requirements for the API -fastapi +fastapi==0.99.1 psutil uvicorn -pydantic \ No newline at end of file +pydantic==1.10.11 \ No newline at end of file From aaed0e49c0c4cf6960d58db7044f7d3709046d71 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Mon, 10 Jul 2023 15:18:09 +0100 Subject: [PATCH 05/32] Add more API methods --- cmd/start => bin/testrun | 16 +++---- framework/python/src/api/api.py | 34 +++++++++++++- framework/python/src/common/session.py | 28 +++++++++++ framework/python/src/core/test_runner.py | 18 ++++---- framework/python/src/core/testrun.py | 46 ++++++++++++++----- .../python/src/test_orc/test_orchestrator.py | 2 +- modules/ui/conf/nginx.conf | 13 ++++++ modules/ui/ui.Dockerfile | 19 ++++++++ ui/index.html | 1 + 9 files changed, 147 insertions(+), 30 deletions(-) rename cmd/start => bin/testrun (76%) create mode 100644 framework/python/src/common/session.py create mode 100644 modules/ui/conf/nginx.conf create mode 100644 modules/ui/ui.Dockerfile create mode 100644 ui/index.html diff --git a/cmd/start b/bin/testrun similarity index 76% rename from cmd/start rename to bin/testrun index 64ac197eb..e991d4082 100755 --- a/cmd/start +++ b/bin/testrun @@ -15,23 +15,23 @@ # limitations under the License. if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo cmd/start" + echo "Must run as root. Use sudo testrun" exit 1 fi -# Ensure that /var/run/netns folder exists -mkdir -p /var/run/netns +# TODO: Obtain TESTRUNPATH from user environment variables +TESTRUNPATH="/home/boddey/Desktop/test-run" +cd $TESTRUNPATH -# Clear up existing runtime files -rm -rf runtime +# Ensure that /var/run/netns folder exists +sudo mkdir -p /var/run/netns -# Check if python modules exist. Install if not -[ ! -d "venv" ] && cmd/install +# Check if Python modules exist. Install if not +[ ! -d "venv" ] && sudo cmd/install # Activate Python virtual environment source venv/bin/activate -# TODO: Execute python code # Set the PYTHONPATH to include the "src" directory export PYTHONPATH="$PWD/framework/python/src" python -u framework/python/src/core/test_runner.py $@ diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 8728d36a1..64b01f5a8 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fastapi import FastAPI, APIRouter +from fastapi import FastAPI, APIRouter, Response, status import json import os import psutil @@ -48,6 +48,12 @@ def __init__(self, test_run): self._router.add_api_route("/system/config", self.post_sys_config, methods=["POST"]) self._router.add_api_route("/system/config", self.get_sys_config) + self._router.add_api_route("/system/start", self.start_test_run, + methods=["POST"]) + self._router.add_api_route("/system/stop", self.stop_test_run, + methods=["POST"]) + self._router.add_api_route("/system/status", self.get_status) + self._router.add_api_route("/devices", self.get_devices) self._app = FastAPI() @@ -60,6 +66,7 @@ def __init__(self, test_run): def start(self): LOGGER.info("Starting API") self._api_thread.start() + LOGGER.info("API waiting for requests") def _start(self): uvicorn.run(self._app, log_config=None) @@ -114,7 +121,7 @@ async def post_sys_config(self, sys_config: SystemConfig): json.dump(json_contents, config_file, indent=2) return sys_config - + async def get_sys_config(self): config_file = open(self._config_file_url, "r", encoding="utf-8") json_contents = json.load(config_file) @@ -123,3 +130,26 @@ async def get_sys_config(self): async def get_devices(self): return self._devices + + async def start_test_run(self, response: Response): + LOGGER.debug("Received start command") + if self._test_run.get_session().status != "Idle": + LOGGER.debug("Test Run is already running. Cannot start another instance.") + response.status_code = status.HTTP_409_CONFLICT + return json.loads('{"error": "Test Run is already running"}') + thread = threading.Thread(target=self._start_test_run, + name="Test Run") + thread.start() + return json.loads('{"status": "Starting Test Run"}') + + def _start_test_run(self): + self._test_run.start() + + async def stop_test_run(self): + LOGGER.info("Received stop command. Stopping Test Run") + + async def get_status(self): + return self._test_run.get_session() + + async def get_history(self): + LOGGER.info("Returning previous Test Runs to UI") diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py new file mode 100644 index 000000000..155c45361 --- /dev/null +++ b/framework/python/src/common/session.py @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Track testing status.""" + +from dataclasses import dataclass, field +from common.device import Device + +@dataclass +class TestRunSession(): + """Represents the current session of Test Run.""" + + status: str = 'Idle' + device: Device = None + started: str = None + finished: str = None + tests: list = field(default_factory=list) diff --git a/framework/python/src/core/test_runner.py b/framework/python/src/core/test_runner.py index 226f874cc..9962c3995 100644 --- a/framework/python/src/core/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -36,12 +36,14 @@ def __init__(self, config_file=None, validate=True, net_only=False, - single_intf=False): + single_intf=False, + no_ui=False): self._register_exits() self.test_run = TestRun(config_file=config_file, validate=validate, net_only=net_only, - single_intf=single_intf) + single_intf=single_intf, + no_ui=no_ui) def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -62,10 +64,6 @@ def _exit_handler(self, signum, arg): # pylint: disable=unused-argument def stop(self, kill=False): self.test_run.stop(kill) - def start(self): - self.test_run.start() - LOGGER.info("Test Run has finished") - def parse_args(): parser = argparse.ArgumentParser( @@ -88,6 +86,10 @@ def parse_args(): parser.add_argument("--single-intf", action="store_true", help="Single interface mode (experimental)") + parser.add_argument("--no-ui", + default=False, + action="store_true", + help="Do not launch the user interface") parsed_args = parser.parse_known_args()[0] return parsed_args @@ -97,5 +99,5 @@ def parse_args(): runner = TestRunner(config_file=args.config_file, validate=not args.no_validate, net_only=args.net_only, - single_intf=args.single_intf) - runner.start() + single_intf=args.single_intf, + no_ui=args.no_ui) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 30f947fab..46c3d1854 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -26,17 +26,19 @@ import time from common import logger from common.device import Device +from common.session import TestRunSession # Locate parent directory current_dir = os.path.dirname(os.path.realpath(__file__)) # Locate the test-run root directory, 4 levels, src->python->framework->test-run -root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) +root_dir = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(current_dir)))) -from api.api import Api -from net_orc.listener import NetworkEvent -from test_orc import test_orchestrator as test_orc -from net_orc import network_orchestrator as net_orc +from api.api import Api # pylint: disable=wrong-import-position +from net_orc.listener import NetworkEvent # pylint: disable=wrong-import-position +from test_orc import test_orchestrator as test_orc # pylint: disable=wrong-import-position +from net_orc import network_orchestrator as net_orc # pylint: disable=wrong-import-position LOGGER = logger.get_logger('test_run') DEFAULT_CONFIG_FILE = 'local/system.json' @@ -54,7 +56,8 @@ def __init__(self, config_file, validate=True, net_only=False, - single_intf=False): + single_intf=False, + no_ui=False): if config_file is None: config_file = DEFAULT_CONFIG_FILE @@ -63,6 +66,9 @@ def __init__(self, self._net_only = net_only self._single_intf = single_intf self._config_file = config_file + self._no_ui = no_ui + + self._session = TestRunSession() # Catch any exit signals self._register_exits() @@ -81,10 +87,14 @@ def __init__(self, self._devices = self._api.load_all_devices() self._api.start() - time.sleep(100) + # Hold until API ends + while True: + time.sleep(1) def start(self): + self._set_status('Starting') + self._start_network() if self._net_only: @@ -115,22 +125,27 @@ def start(self): ) self._net_orc.start_listener() + self._set_status('Waiting for device') LOGGER.info('Waiting for devices on the network...') time.sleep(RUNTIME) - if not (self._test_orc.test_in_progress() or self._net_orc.monitor_in_progress()): - LOGGER.info('Timed out whilst waiting for device or stopping due to test completion') + if not (self._test_orc.test_in_progress() or + self._net_orc.monitor_in_progress()): + LOGGER.info('''Timed out whilst waiting for + device or stopping due to test completion''') else: - while self._test_orc.test_in_progress() or self._net_orc.monitor_in_progress(): + while (self._test_orc.test_in_progress() or + self._net_orc.monitor_in_progress()): time.sleep(5) self.stop() def stop(self, kill=False): + self._set_status('Stopping') self._stop_tests() self._stop_network(kill=kill) - self._api.stop() + self._set_status('Idle') def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -175,6 +190,7 @@ def get_device(self, mac_addr): return device def _device_discovered(self, mac_addr): + self._set_status('Identifying device') device = self.get_device(mac_addr) if device is not None: LOGGER.info( @@ -188,4 +204,12 @@ def _device_discovered(self, mac_addr): def _device_stable(self, mac_addr): device = self.get_device(mac_addr) LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') + self._set_status('In progress') self._test_orc.run_test_modules(device) + self._set_status('Complete') + + def _set_status(self, status): + self._session.status = status + + def get_session(self): + return self._session diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 4bc9fc003..037dbb650 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -110,7 +110,7 @@ def _generate_results(self, device): "runtime/test/" + device.mac_addr.replace(":", "") + "/results.json") with open(out_file, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) - util.run_command(f'chown -R {self._host_user} {out_file}') + util.run_command(f"chown -R {self._host_user} {out_file}") return results def test_in_progress(self): diff --git a/modules/ui/conf/nginx.conf b/modules/ui/conf/nginx.conf new file mode 100644 index 000000000..ade6ad17a --- /dev/null +++ b/modules/ui/conf/nginx.conf @@ -0,0 +1,13 @@ +events{} +http { + include /etc/nginx/mime.types; + server { + listen 80; + server_name localhost; + root /usr/share/nginx/html; + index index.html; + location / { + try_files $uri $uri/ /index.html; + } + } +} \ No newline at end of file diff --git a/modules/ui/ui.Dockerfile b/modules/ui/ui.Dockerfile new file mode 100644 index 000000000..f65f4c48b --- /dev/null +++ b/modules/ui/ui.Dockerfile @@ -0,0 +1,19 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Image name: test-run/ui +FROM nginx:1.25.1 + +COPY modules/ui/conf/nginx.conf /etc/nginx/nginx.conf +COPY ui /usr/share/nginx/html \ No newline at end of file diff --git a/ui/index.html b/ui/index.html new file mode 100644 index 000000000..285fce5ad --- /dev/null +++ b/ui/index.html @@ -0,0 +1 @@ +Test Run \ No newline at end of file From 3d1c1ae28616e819b9d59e247def6fcba6ad5d36 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Mon, 10 Jul 2023 15:36:43 +0100 Subject: [PATCH 06/32] Correct no-ui flag --- bin/testrun | 4 ++-- framework/python/src/core/testrun.py | 9 ++++++--- testing/test_baseline | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/testrun b/bin/testrun index e991d4082..fde14451f 100755 --- a/bin/testrun +++ b/bin/testrun @@ -20,8 +20,8 @@ if [[ "$EUID" -ne 0 ]]; then fi # TODO: Obtain TESTRUNPATH from user environment variables -TESTRUNPATH="/home/boddey/Desktop/test-run" -cd $TESTRUNPATH +#TESTRUNPATH="/home/boddey/Desktop/test-run" +#cd $TESTRUNPATH # Ensure that /var/run/netns folder exists sudo mkdir -p /var/run/netns diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 46c3d1854..5421b00dd 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -83,9 +83,12 @@ def __init__(self, self._test_orc = test_orc.TestOrchestrator(self._net_orc) - self._api = Api(self) - self._devices = self._api.load_all_devices() - self._api.start() + if self._no_ui: + self.start() + else: + self._api = Api(self) + self._devices = self._api.load_all_devices() + self._api.start() # Hold until API ends while True: diff --git a/testing/test_baseline b/testing/test_baseline index ac47a5cfa..25c6d4ef8 100755 --- a/testing/test_baseline +++ b/testing/test_baseline @@ -48,7 +48,7 @@ EOF sudo cmd/install -sudo cmd/start --single-intf > $TESTRUN_OUT 2>&1 & +sudo bin/testrun --single-intf > $TESTRUN_OUT 2>&1 & TPID=$! # Time to wait for testrun to be ready From 2f17fced3e301c3d351082b0ab5da8dad4aa31dd Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Mon, 10 Jul 2023 15:46:52 +0100 Subject: [PATCH 07/32] Do not launch API on baseline test --- testing/test_baseline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_baseline b/testing/test_baseline index 25c6d4ef8..64c39e397 100755 --- a/testing/test_baseline +++ b/testing/test_baseline @@ -48,7 +48,7 @@ EOF sudo cmd/install -sudo bin/testrun --single-intf > $TESTRUN_OUT 2>&1 & +sudo bin/testrun --single-intf --no-ui > $TESTRUN_OUT 2>&1 & TPID=$! # Time to wait for testrun to be ready From 7c4fcda8fce248cd6186893b7554dc2f900754e6 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 12 Jul 2023 11:17:47 +0100 Subject: [PATCH 08/32] Move loading devices back to Test Run core --- framework/python/src/api/api.py | 40 ++------------------------ framework/python/src/core/testrun.py | 43 ++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 64b01f5a8..40cda24be 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -14,23 +14,14 @@ from fastapi import FastAPI, APIRouter, Response, status import json -import os import psutil import threading import uvicorn from api.system_config import SystemConfig -from common import logger, util -from common.device import Device +from common import logger LOGGER = logger.get_logger("api") -LOCAL_DEVICES_DIR = "local/devices" -RESOURCE_DEVICES_DIR = "resources/devices" -DEVICE_CONFIG = "device_config.json" -DEVICE_MANUFACTURER = "manufacturer" -DEVICE_MODEL = "model" -DEVICE_MAC_ADDR = "mac_addr" -DEVICE_TEST_MODULES = "test_modules" class Api: """Provide REST endpoints to manage Test Run""" @@ -41,7 +32,7 @@ def __init__(self, test_run): self._name = "TestRun API" self._router = APIRouter() - self._devices = [] + self._devices = self._test_run.get_devices() self._config_file_url = self._test_run.get_config_file() self._router.add_api_route("/system/interfaces", self.get_sys_interfaces) @@ -74,33 +65,6 @@ def _start(self): def stop(self): LOGGER.info("Stopping API") - def load_all_devices(self): - self._load_devices(device_dir=LOCAL_DEVICES_DIR) - self._load_devices(device_dir=RESOURCE_DEVICES_DIR) - return self._devices - - def _load_devices(self, device_dir): - LOGGER.debug("Loading devices from " + device_dir) - - os.makedirs(device_dir, exist_ok=True) - util.run_command(f"chown -R {util.get_host_user()} {device_dir}") - - for device_folder in os.listdir(device_dir): - with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), - encoding="utf-8") as device_config_file: - device_config_json = json.load(device_config_file) - - device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) - device_model = device_config_json.get(DEVICE_MODEL) - mac_addr = device_config_json.get(DEVICE_MAC_ADDR) - test_modules = device_config_json.get(DEVICE_TEST_MODULES) - - device = Device(manufacturer=device_manufacturer, - model=device_model, - mac_addr=mac_addr, - test_modules=test_modules) - self._devices.append(device) - async def get_sys_interfaces(self): addrs = psutil.net_if_addrs() ifaces = [] diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 5421b00dd..4f424a185 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -20,11 +20,12 @@ Run using the provided command scripts in the cmd folder. E.g sudo cmd/start """ +import json import os import sys import signal import time -from common import logger +from common import logger, util from common.device import Device from common.session import TestRunSession @@ -44,6 +45,13 @@ DEFAULT_CONFIG_FILE = 'local/system.json' EXAMPLE_CONFIG_FILE = 'local/system.json.example' RUNTIME = 120 +LOCAL_DEVICES_DIR = 'local/devices' +RESOURCE_DEVICES_DIR = 'resources/devices' +DEVICE_CONFIG = 'device_config.json' +DEVICE_MANUFACTURER = 'manufacturer' +DEVICE_MODEL = 'model' +DEVICE_MAC_ADDR = 'mac_addr' +DEVICE_TEST_MODULES = 'test_modules' class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. @@ -76,6 +84,8 @@ def __init__(self, # Expand the config file to absolute pathing config_file_abs = self._get_config_abs(config_file=self._config_file) + self._load_all_devices() + self._net_orc = net_orc.NetworkOrchestrator( config_file=config_file_abs, validate=validate, @@ -87,13 +97,39 @@ def __init__(self, self.start() else: self._api = Api(self) - self._devices = self._api.load_all_devices() self._api.start() # Hold until API ends while True: time.sleep(1) + def _load_all_devices(self): + self._load_devices(device_dir=LOCAL_DEVICES_DIR) + self._load_devices(device_dir=RESOURCE_DEVICES_DIR) + return self._devices + + def _load_devices(self, device_dir): + LOGGER.debug('Loading devices from ' + device_dir) + + os.makedirs(device_dir, exist_ok=True) + util.run_command(f'chown -R {util.get_host_user()} {device_dir}') + + for device_folder in os.listdir(device_dir): + with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), + encoding='utf-8') as device_config_file: + device_config_json = json.load(device_config_file) + + device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) + device_model = device_config_json.get(DEVICE_MODEL) + mac_addr = device_config_json.get(DEVICE_MAC_ADDR) + test_modules = device_config_json.get(DEVICE_TEST_MODULES) + + device = Device(manufacturer=device_manufacturer, + model=device_model, + mac_addr=mac_addr, + test_modules=test_modules) + self._devices.append(device) + def start(self): self._set_status('Starting') @@ -174,6 +210,9 @@ def _get_config_abs(self, config_file=None): def get_config_file(self): return self._get_config_abs() + def get_devices(self): + return self._devices + def _start_network(self): # Start the network orchestrator if not self._net_orc.start(): From 3d244af585afef898c1e4a16483107fc9b61d5ab Mon Sep 17 00:00:00 2001 From: J Boddey Date: Fri, 14 Jul 2023 11:33:27 +0100 Subject: [PATCH 09/32] Merge dev into api (#74) * Merge dev into main (Add license header) (#62) Add license header * Add network docs (#63) * Add network docs * Rename to readme * Add link to template module * Dhcp (#64) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * initial work on adding grpc functions for dhcp tests * rework code to allow for better usage and unit testing * working poc for test containers and grpc client to dhcp-1 * Move grpc client code into base image * Move grpc proto builds outside of dockerfile into module startup script * Setup pythonpath var in test module base startup process misc cleanup * pylinting and logging updates * Add python path resolving to network modules Update grpc path to prevent conflicts misc pylinting * Change lease resolving method to fix pylint issue * cleanup unit tests * cleanup unit tests * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * fix line endings * misc cleanup * Dhcp (#67) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * initial work on adding grpc functions for dhcp tests * rework code to allow for better usage and unit testing * working poc for test containers and grpc client to dhcp-1 * Move grpc client code into base image * Move grpc proto builds outside of dockerfile into module startup script * Setup pythonpath var in test module base startup process misc cleanup * pylinting and logging updates * Add python path resolving to network modules Update grpc path to prevent conflicts misc pylinting * Change lease resolving method to fix pylint issue * cleanup unit tests * cleanup unit tests * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * fix line endings * misc cleanup * Move isc-dhcp-server and radvd to services Move DHCP server monitoring and booting to python script * Add grpc methods to interact with dhcp_server module Update dhcp_server to control radvd server directly from calls Fix radvd service status method * Add updates to dhcp2 module Update radvd service * Add license headers * Add connection.dhcp_address test (#68) * Add NTP tests (#60) * Add ntp support test * Add extra log message * Modify descriptions * Pylint * Pylint (#69) --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> * Add ipv6 tests (#65) * Add ipv6 tests * Check for ND_NS * Merge dev into main (Sprint 9) (#66) * Implement test orchestrator (#4) * Initial work on test-orchestrator * Ignore runtime folder * Update runtime directory for test modules * Fix logging Add initial framework for running tests * logging and misc cleanup * logging changes * Add a stop hook after all tests complete * Refactor test_orc code * Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator * Fix baseline module Fix orchestrator exiting only after timeout * Add result file to baseline test module Change result format to match closer to design doc * Refactor pylint * Skip test module if it failed to start * Refactor * Check for valid log level --------- Co-authored-by: Jacob Boddey * Add issue report templates (#7) * Add issue templates * Update README.md * Discover devices on the network (#5) * Test run sync (#8) * Initial work on test-orchestrator * Ignore runtime folder * Update runtime directory for test modules * Fix logging Add initial framework for running tests * logging and misc cleanup * logging changes * Add a stop hook after all tests complete * Refactor test_orc code * Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator * Fix baseline module Fix orchestrator exiting only after timeout * Add result file to baseline test module Change result format to match closer to design doc * Refactor pylint * Skip test module if it failed to start * Refactor * Check for valid log level * Add config file arg Misc changes to network start procedure * fix merge issues * Update runner and test orch procedure Add useful runtiem args * Restructure test run startup process Misc updates to work with net orch updates * Refactor --------- * Quick refactor (#9) * Fix duplicate sleep calls * Add net orc (#11) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add the DNS test module (#12) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add dns test module Fix test module build process * Add mac address of device under test to test container Update dns test to use mac address filter * Update dns module tests * Change result output * logging update * Update test module for better reusability * Load in module config to test module * logging cleanup * Update baseline module to new template Misc cleanup * Add ability to disable individual tests * remove duplicate readme * Update device directories * Remove local folder * Update device template Update test module to work with new device config file format * Change test module network config options Do not start network services for modules not configured for network * Refactor --------- * Add baseline and pylint tests (#25) * Discover devices on the network (#22) * Discover devices on the network * Add defaults when missing from config Implement monitor wait period from config * Add steady state monitor Remove duplicate callback registrations * Load devices into network orchestrator during testrun start --------- Co-authored-by: jhughesbiot * Build dependencies first (#21) * Build dependencies first * Remove debug message * Add depend on option to test modules * Re-add single interface option * Import subprocess --------- Co-authored-by: jhughesbiot * Port scan test module (#23) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add dns test module Fix test module build process * Add mac address of device under test to test container Update dns test to use mac address filter * Update dns module tests * Change result output * logging update * Update test module for better reusability * Load in module config to test module * logging cleanup * Update baseline module to new template Misc cleanup * Add ability to disable individual tests * remove duplicate readme * Update device directories * Remove local folder * Update device template Update test module to work with new device config file format * Change test module network config options Do not start network services for modules not configured for network * Initial nmap test module add Add device ip resolving to base module Add network mounting for test modules * Update ipv4 device resolving in test modules * Map in ip subnets and remove hard coded references * Add ftp port test * Add ability to pass config for individual tests within a module Update nmap module scan to run tests based on config * Add full module check for compliance * Add all tcp port scans to config * Update nmap commands to match existing DAQ tests Add udp scanning and tests * logging cleanup * Update TCP port scanning range Update logging * Merge device config into module config Update device template * fix merge issues * Update timeouts Add multi-threading for multiple scanns to run simultaneously Add option to use scan scripts for services * Fix merge issues * Fix device configs * Remove unecessary files * Cleanup duplicate properties * Cleanup install script * Formatting (#26) * Fix pylint issues in net orc * more pylint fixes * fix listener lint issues * fix logger lint issues * fix validator lint issues * fix util lint issues * Update base network module linting issues * Cleanup linter issues for dhcp modules Remove old code testing code * change to single quote delimeter * Cleanup linter issues for ntp module * Cleanup linter issues for radius module * Cleanup linter issues for template module * fix linter issues with faux-dev * Test results (#27) * Collect all module test results * Fix test modules without config options * Add timestamp to test results * Test results (#28) * Collect all module test results * Fix test modules without config options * Add timestamp to test results * Add attempt timing and device info to test results * Ignore disabled test containers when generating results * Fully skip modules that are disabled * Fix pylint test and skip internet tests so CI passes (#29) * disable internet checks for pass * fix pylint test * Increase pylint score (#31) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger --------- Co-authored-by: jhughesbiot * Pylint (#32) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting --------- Co-authored-by: Jacob Boddey * Add license header (#36) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module * add license header to all python files --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * Ovs (#35) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * remove ovs files added back in during merge * Nmap (#38) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module * add license header to all python files * Update tcp scans to speed up full port range scan Add version checking Implement ssh version checking * Add unknown port checks Match unknown ports to existing services Add unknown ports without existing services to results file --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * Create startup capture (#37) * Connection (#40) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Conn mac oui (#42) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Add mac_oui test Add option to return test result and details of test for reporting * Con mac address (#43) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Add mac_oui test Add option to return test result and details of test for reporting * Add connection.mac_address test * Dns (#44) * Add MDNS test * Update existing mdns logging to be more consistent with other tests * Add startup and monitor captures * File permissions (#45) * Fix validator file permissions * Fix test module permissions * Fix device capture file permissions * Fix device results permissions * Add connection single ip test (#47) * Nmap results (#49) * Update processing of nmap results to use xml output and json conversions for stability * Update matching with regex to prevent wrong service matches and duplicate processing for partial matches * Update max port scan range * Framework restructure (#50) * Restructure framework and modules * Fix CI paths * Fix base module * Add build script * Remove build logs * Update base and template docker files to fit the new format Implement a template option on network modules Fix skipping of base image build * remove base image build in ci * Remove group from chown --------- Co-authored-by: jhughesbiot * Ip control (#51) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * Move config to /local (#52) * Move config to /local * Fix testing config * Fix ovs_control config location * Fix faux dev config location * Add documentation (#53) * Sync dev to main (#56) * Merge dev into main (Sprint 7 and 8) (#33) * Implement test orchestrator (#4) * Initial work on test-orchestrator * Ignore runtime folder * Update runtime directory for test modules * Fix logging Add initial framework for running tests * logging and misc cleanup * logging changes * Add a stop hook after all tests complete * Refactor test_orc code * Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator * Fix baseline module Fix orchestrator exiting only after timeout * Add result file to baseline test module Change result format to match closer to design doc * Refactor pylint * Skip test module if it failed to start * Refactor * Check for valid log level --------- Co-authored-by: Jacob Boddey * Add issue report templates (#7) * Add issue templates * Update README.md * Discover devices on the network (#5) * Test run sync (#8) * Initial work on test-orchestrator * Ignore runtime folder * Update runtime directory for test modules * Fix logging Add initial framework for running tests * logging and misc cleanup * logging changes * Add a stop hook after all tests complete * Refactor test_orc code * Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator * Fix baseline module Fix orchestrator exiting only after timeout * Add result file to baseline test module Change result format to match closer to design doc * Refactor pylint * Skip test module if it failed to start * Refactor * Check for valid log level * Add config file arg Misc changes to network start procedure * fix merge issues * Update runner and test orch procedure Add useful runtiem args * Restructure test run startup process Misc updates to work with net orch updates * Refactor --------- * Quick refactor (#9) * Fix duplicate sleep calls * Add net orc (#11) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add the DNS test module (#12) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add dns test module Fix test module build process * Add mac address of device under test to test container Update dns test to use mac address filter * Update dns module tests * Change result output * logging update * Update test module for better reusability * Load in module config to test module * logging cleanup * Update baseline module to new template Misc cleanup * Add ability to disable individual tests * remove duplicate readme * Update device directories * Remove local folder * Update device template Update test module to work with new device config file format * Change test module network config options Do not start network services for modules not configured for network * Refactor --------- * Add baseline and pylint tests (#25) * Discover devices on the network (#22) * Discover devices on the network * Add defaults when missing from config Implement monitor wait period from config * Add steady state monitor Remove duplicate callback registrations * Load devices into network orchestrator during testrun start --------- Co-authored-by: jhughesbiot * Build dependencies first (#21) * Build dependencies first * Remove debug message * Add depend on option to test modules * Re-add single interface option * Import subprocess --------- Co-authored-by: jhughesbiot * Port scan test module (#23) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add dns test module Fix test module build process * Add mac address of device under test to test container Update dns test to use mac address filter * Update dns module tests * Change result output * logging update * Update test module for better reusability * Load in module config to test module * logging cleanup * Update baseline module to new template Misc cleanup * Add ability to disable individual tests * remove duplicate readme * Update device directories * Remove local folder * Update device template Update test module to work with new device config file format * Change test module network config options Do not start network services for modules not configured for network * Initial nmap test module add Add device ip resolving to base module Add network mounting for test modules * Update ipv4 device resolving in test modules * Map in ip subnets and remove hard coded references * Add ftp port test * Add ability to pass config for individual tests within a module Update nmap module scan to run tests based on config * Add full module check for compliance * Add all tcp port scans to config * Update nmap commands to match existing DAQ tests Add udp scanning and tests * logging cleanup * Update TCP port scanning range Update logging * Merge device config into module config Update device template * fix merge issues * Update timeouts Add multi-threading for multiple scanns to run simultaneously Add option to use scan scripts for services * Fix merge issues * Fix device configs * Remove unecessary files * Cleanup duplicate properties * Cleanup install script * Formatting (#26) * Fix pylint issues in net orc * more pylint fixes * fix listener lint issues * fix logger lint issues * fix validator lint issues * fix util lint issues * Update base network module linting issues * Cleanup linter issues for dhcp modules Remove old code testing code * change to single quote delimeter * Cleanup linter issues for ntp module * Cleanup linter issues for radius module * Cleanup linter issues for template module * fix linter issues with faux-dev * Test results (#27) * Collect all module test results * Fix test modules without config options * Add timestamp to test results * Test results (#28) * Collect all module test results * Fix test modules without config options * Add timestamp to test results * Add attempt timing and device info to test results * Ignore disabled test containers when generating results * Fully skip modules that are disabled * Fix pylint test and skip internet tests so CI passes (#29) * disable internet checks for pass * fix pylint test * Increase pylint score (#31) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger --------- Co-authored-by: jhughesbiot * Pylint (#32) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting --------- Co-authored-by: Jacob Boddey * Add license header (#36) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module * add license header to all python files --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * Ovs (#35) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * remove ovs files added back in during merge * Nmap (#38) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module * add license header to all python files * Update tcp scans to speed up full port range scan Add version checking Implement ssh version checking * Add unknown port checks Match unknown ports to existing services Add unknown ports without existing services to results file --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * Create startup capture (#37) * Connection (#40) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Conn mac oui (#42) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Add mac_oui test Add option to return test result and details of test for reporting * Con mac address (#43) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Add mac_oui test Add option to return test result and details of test for reporting * Add connection.mac_address test * Dns (#44) * Add MDNS test * Update existing mdns logging to be more consistent with other tests * Add startup and monitor captures * File permissions (#45) * Fix validator file permissions * Fix test module permissions * Fix device capture file permissions * Fix device results permissions * Add connection single ip test (#47) * Nmap results (#49) * Update processing of nmap results to use xml output and json conversions for stability * Update matching with regex to prevent wrong service matches and duplicate processing for partial matches * Update max port scan range * Framework restructure (#50) * Restructure framework and modules * Fix CI paths * Fix base module * Add build script * Remove build logs * Update base and template docker files to fit the new format Implement a template option on network modules Fix skipping of base image build * remove base image build in ci * Remove group from chown --------- Co-authored-by: jhughesbiot * Ip control (#51) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * Move config to /local (#52) * Move config to /local * Fix testing config * Fix ovs_control config location * Fix faux dev config location * Add documentation (#53) --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> Co-authored-by: jhughesbiot Co-authored-by: Noureddine Co-authored-by: SuperJonotron * Sprint 8 Hotfix (#54) * Fix connection results.json * Re add try/catch * Fix log level * Debug test module load order * Add depends on to nmap module * Remove logging change --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> Co-authored-by: jhughesbiot Co-authored-by: Noureddine Co-authored-by: SuperJonotron * Fix missing results on udp tests when tcp ports are also defined (#59) * Add licence header (#61) * Resolve merge conflict * Add network docs (#63) * Add network docs * Rename to readme * Add link to template module * Dhcp (#64) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * initial work on adding grpc functions for dhcp tests * rework code to allow for better usage and unit testing * working poc for test containers and grpc client to dhcp-1 * Move grpc client code into base image * Move grpc proto builds outside of dockerfile into module startup script * Setup pythonpath var in test module base startup process misc cleanup * pylinting and logging updates * Add python path resolving to network modules Update grpc path to prevent conflicts misc pylinting * Change lease resolving method to fix pylint issue * cleanup unit tests * cleanup unit tests * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * fix line endings * misc cleanup * Dhcp (#67) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * initial work on adding grpc functions for dhcp tests * rework code to allow for better usage and unit testing * working poc for test containers and grpc client to dhcp-1 * Move grpc client code into base image * Move grpc proto builds outside of dockerfile into module startup script * Setup pythonpath var in test module base startup process misc cleanup * pylinting and logging updates * Add python path resolving to network modules Update grpc path to prevent conflicts misc pylinting * Change lease resolving method to fix pylint issue * cleanup unit tests * cleanup unit tests * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables * fix line endings * misc cleanup * Move isc-dhcp-server and radvd to services Move DHCP server monitoring and booting to python script * Add grpc methods to interact with dhcp_server module Update dhcp_server to control radvd server directly from calls Fix radvd service status method * Add updates to dhcp2 module Update radvd service * Add license headers * Add connection.dhcp_address test (#68) * Add NTP tests (#60) * Add ntp support test * Add extra log message * Modify descriptions * Pylint * Pylint (#69) --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> * Add ipv6 tests (#65) * Add ipv6 tests * Check for ND_NS --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> Co-authored-by: jhughesbiot Co-authored-by: Noureddine Co-authored-by: SuperJonotron * Connection private address (#71) * Add ntp support test * Add extra log message * Modify descriptions * Pylint * formatting * Change isc-dhcp service setup Fix dhcpd logging Add start and stop methods to grpc dhcp client Add dhcp2 client Inttial private_addr test * Add max lease time Add unit tests * fix last commit * finish initial work on test * pylinting * Breakup test and allow better failure reporting * restore network after test * Wait for device to get a lease from original dhcp range after network restore * pylinting * Fix ipv6 tests --------- Co-authored-by: Jacob Boddey * fix windows line ending * Fix python import * move isc-dhcp service commands to their own class update logging pylinting * fix dhcp1 * Initial CI testing for tests (#72) * Fix radvd conf --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> Co-authored-by: jhughesbiot Co-authored-by: Noureddine Co-authored-by: SuperJonotron --- .github/workflows/testing.yml | 18 +- docs/network/README.md | 41 ++ docs/network/add_new_service.md | 94 +++ docs/network/addresses.md | 18 + .../python/src/net_orc/network_validator.py | 12 +- .../python/src/test_orc/test_orchestrator.py | 15 +- local/system.json.example | 18 +- modules/network/base/base.Dockerfile | 4 + modules/network/base/bin/setup_python_path | 25 + modules/network/base/bin/start_grpc | 6 +- modules/network/base/bin/start_module | 18 +- modules/network/base/python/requirements.txt | 3 +- .../src/{grpc => grpc_server}/start_server.py | 0 modules/network/base/python/src/logger.py | 2 +- modules/network/dhcp-1/bin/isc-dhcp-service | 56 ++ modules/network/dhcp-1/bin/radvd-service | 55 ++ .../network/dhcp-1/bin/start_network_service | 60 +- modules/network/dhcp-1/conf/dhcpd.conf | 31 +- modules/network/dhcp-1/conf/isc-dhcp-server | 4 + .../network/dhcp-1/conf/module_config.json | 50 +- modules/network/dhcp-1/conf/radvd.conf | 4 +- modules/network/dhcp-1/dhcp-1.Dockerfile | 10 +- .../dhcp-1/python/src/grpc/dhcp_config.py | 303 ---------- .../dhcp-1/python/src/grpc/network_service.py | 58 -- .../dhcp-1/python/src/grpc/proto/grpc.proto | 36 -- .../src/{grpc => grpc_server}/__init__.py | 0 .../python/src/grpc_server/dhcp_config.py | 551 +++++++++++++++++ .../src/grpc_server/dhcp_config_test.py | 116 ++++ .../python/src/grpc_server/dhcp_lease.py | 75 +++ .../python/src/grpc_server/dhcp_leases.py | 107 ++++ .../python/src/grpc_server/dhcp_server.py | 107 ++++ .../python/src/grpc_server/isc_dhcp_server.py | 52 ++ .../python/src/grpc_server/network_service.py | 195 ++++++ .../python/src/grpc_server/proto/grpc.proto | 71 +++ .../python/src/grpc_server/radvd_server.py | 52 ++ modules/network/dhcp-2/bin/isc-dhcp-service | 56 ++ modules/network/dhcp-2/bin/radvd-service | 55 ++ .../network/dhcp-2/bin/start_network_service | 59 +- modules/network/dhcp-2/conf/dhcpd.conf | 50 +- modules/network/dhcp-2/conf/isc-dhcp-server | 4 + modules/network/dhcp-2/dhcp-2.Dockerfile | 11 +- .../dhcp-2/python/src/grpc/dhcp_config.py | 303 ---------- .../dhcp-2/python/src/grpc/network_service.py | 58 -- .../dhcp-2/python/src/grpc/proto/grpc.proto | 36 -- .../src/{grpc => grpc_server}/__init__.py | 0 .../python/src/grpc_server/dhcp_config.py | 553 ++++++++++++++++++ .../src/grpc_server/dhcp_config_test.py | 112 ++++ .../python/src/grpc_server/dhcp_lease.py | 75 +++ .../python/src/grpc_server/dhcp_leases.py | 107 ++++ .../python/src/grpc_server/dhcp_server.py | 106 ++++ .../python/src/grpc_server/isc_dhcp_server.py | 52 ++ .../python/src/grpc_server/network_service.py | 195 ++++++ .../python/src/grpc_server/proto/grpc.proto | 71 +++ .../python/src/grpc_server/radvd_server.py | 52 ++ modules/test/base/base.Dockerfile | 8 + modules/test/base/bin/setup_grpc_clients | 34 ++ modules/test/base/bin/setup_python_path | 25 + modules/test/base/bin/start_module | 17 +- .../python/src/grpc/proto/dhcp1/client.py | 131 +++++ .../python/src/grpc/proto/dhcp2/client.py | 130 ++++ modules/test/base/python/src/test_module.py | 18 +- modules/test/conn/conf/module_config.json | 36 +- modules/test/conn/conn.Dockerfile | 4 +- .../test/conn/python/src/connection_module.py | 390 ++++++++++-- modules/test/nmap/nmap.Dockerfile | 4 +- modules/test/ntp/bin/start_test_module | 42 ++ modules/test/ntp/conf/module_config.json | 27 + modules/test/ntp/ntp.Dockerfile | 20 + modules/test/ntp/python/requirements.txt | 1 + modules/test/ntp/python/src/ntp_module.py | 79 +++ modules/test/ntp/python/src/run.py | 75 +++ resources/devices/template/device_config.json | 29 + testing/docker/ci_test_device1/Dockerfile | 11 + testing/docker/ci_test_device1/entrypoint.sh | 91 +++ testing/example/mac | 0 testing/example/mac1/results.json | 252 ++++++++ testing/test_baseline | 6 +- testing/test_baseline.py | 13 +- testing/test_pylint | 4 +- testing/test_tests | 120 ++++ testing/test_tests.json | 19 + testing/test_tests.py | 102 ++++ testing/unit_test/run_tests.sh | 18 + 83 files changed, 4863 insertions(+), 1065 deletions(-) create mode 100644 docs/network/README.md create mode 100644 docs/network/add_new_service.md create mode 100644 docs/network/addresses.md create mode 100644 modules/network/base/bin/setup_python_path rename modules/network/base/python/src/{grpc => grpc_server}/start_server.py (100%) create mode 100644 modules/network/dhcp-1/bin/isc-dhcp-service create mode 100644 modules/network/dhcp-1/bin/radvd-service create mode 100644 modules/network/dhcp-1/conf/isc-dhcp-server delete mode 100644 modules/network/dhcp-1/python/src/grpc/dhcp_config.py delete mode 100644 modules/network/dhcp-1/python/src/grpc/network_service.py delete mode 100644 modules/network/dhcp-1/python/src/grpc/proto/grpc.proto rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/__init__.py (100%) create mode 100644 modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/dhcp_lease.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/isc_dhcp_server.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/network_service.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto create mode 100644 modules/network/dhcp-1/python/src/grpc_server/radvd_server.py create mode 100644 modules/network/dhcp-2/bin/isc-dhcp-service create mode 100644 modules/network/dhcp-2/bin/radvd-service create mode 100644 modules/network/dhcp-2/conf/isc-dhcp-server delete mode 100644 modules/network/dhcp-2/python/src/grpc/dhcp_config.py delete mode 100644 modules/network/dhcp-2/python/src/grpc/network_service.py delete mode 100644 modules/network/dhcp-2/python/src/grpc/proto/grpc.proto rename modules/network/dhcp-2/python/src/{grpc => grpc_server}/__init__.py (100%) create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/isc_dhcp_server.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/network_service.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto create mode 100644 modules/network/dhcp-2/python/src/grpc_server/radvd_server.py create mode 100644 modules/test/base/bin/setup_grpc_clients create mode 100644 modules/test/base/bin/setup_python_path create mode 100644 modules/test/base/python/src/grpc/proto/dhcp1/client.py create mode 100644 modules/test/base/python/src/grpc/proto/dhcp2/client.py create mode 100644 modules/test/ntp/bin/start_test_module create mode 100644 modules/test/ntp/conf/module_config.json create mode 100644 modules/test/ntp/ntp.Dockerfile create mode 100644 modules/test/ntp/python/requirements.txt create mode 100644 modules/test/ntp/python/src/ntp_module.py create mode 100644 modules/test/ntp/python/src/run.py create mode 100644 testing/docker/ci_test_device1/Dockerfile create mode 100755 testing/docker/ci_test_device1/entrypoint.sh create mode 100644 testing/example/mac create mode 100644 testing/example/mac1/results.json create mode 100755 testing/test_tests create mode 100644 testing/test_tests.json create mode 100644 testing/test_tests.py create mode 100644 testing/unit_test/run_tests.sh diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index fbdbe442c..c981dbd56 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -7,7 +7,7 @@ on: - cron: '0 13 * * *' jobs: - testrun: + testrun_baseline: name: Baseline runs-on: ubuntu-20.04 timeout-minutes: 20 @@ -17,11 +17,21 @@ jobs: - name: Run tests shell: bash {0} run: testing/test_baseline - + + testrun_tests: + name: Tests + runs-on: ubuntu-20.04 + timeout-minutes: 40 + steps: + - name: Checkout source + uses: actions/checkout@v2.3.4 + - name: Run tests + shell: bash {0} + run: testing/test_tests pylint: name: Pylint - runs-on: ubuntu-20.04 - timeout-minutes: 20 + runs-on: ubuntu-22.04 + timeout-minutes: 5 steps: - name: Checkout source uses: actions/checkout@v2.3.4 diff --git a/docs/network/README.md b/docs/network/README.md new file mode 100644 index 000000000..2d66d3e6a --- /dev/null +++ b/docs/network/README.md @@ -0,0 +1,41 @@ +# Network Overview + +## Table of Contents +1) Network Overview (this page) +2) [Addresses](addresses.md) +3) [Add a new network service](add_new_service.md) + +Test Run provides several built-in network services that can be utilized for testing purposes. These services are already available and can be used without any additional configuration. + +The following network services are provided: + +### Internet Connectivity (Gateway Service) + +The gateway service provides internet connectivity to the test network. It allows devices in the network to access external resources and communicate with the internet. + +### DHCPv4 Service + +The DHCPv4 service provides Dynamic Host Configuration Protocol (DHCP) functionality for IPv4 addressing. It includes the following components: + +- Primary DHCP Server: A primary DHCP server is available to assign IPv4 addresses to DHCP clients in the network. +- Secondary DHCP Server (Failover Configuration): A secondary DHCP server operates in failover configuration with the primary server to provide high availability and redundancy. + +#### Configuration + +The configuration of the DHCPv4 service can be modified using the provided GRPC (gRPC Remote Procedure Call) service. + +### IPv6 SLAAC Addressing + +The primary DHCP server also provides IPv6 Stateless Address Autoconfiguration (SLAAC) addressing for devices in the network. IPv6 addresses are automatically assigned to devices using SLAAC where test devices support it. + +### NTP Service + +The Network Time Protocol (NTP) service provides time synchronization for devices in the network. It ensures that all devices have accurate and synchronized time information. + +### DNS Service + +The DNS (Domain Name System) service resolves domain names to their corresponding IP addresses. It allows devices in the network to access external resources using domain names. + +### 802.1x Authentication (Radius Module) + +The radius module provides 802.1x authentication for devices in the network. It ensures secure and authenticated access to the network. The issuing CA (Certificate Authority) certificate can be specified by the user if required. \ No newline at end of file diff --git a/docs/network/add_new_service.md b/docs/network/add_new_service.md new file mode 100644 index 000000000..1ad07b60d --- /dev/null +++ b/docs/network/add_new_service.md @@ -0,0 +1,94 @@ +# Adding a New Network Service + +The Test Run framework allows users to add their own network services with ease. A template network service can be used to get started quickly, this can be found at [modules/network/template](../../modules/network/template). Otherwise, see below for details of the requirements for new network services. + +To add a new network service to Test Run, follow the procedure below: + +1. Create a folder under `modules/network/` with the name of the network service in lowercase, using only alphanumeric characters and hyphens (`-`). +2. Inside the created folder, include the following files and folders: + - `{module}.Dockerfile`: Dockerfile for building the network service image. Replace `{module}` with the name of the module. + - `conf/`: Folder containing the module configuration files. + - `bin/`: Folder containing the startup script for the network service. + - Any additional application code can be placed in its own folder. + +### Example `module_config.json` + +```json +{ + "config": { + "meta": { + "name": "{module}", + "display_name": "Network Service Name", + "description": "Description of the network service" + }, + "network": { + "interface": "veth0", + "enable_wan": false, + "ip_index": 2 + }, + "grpc": { + "port": 5001 + }, + "docker": { + "depends_on": "base", + "mounts": [ + { + "source": "runtime/network", + "target": "/runtime/network" + } + ] + } + } +} +``` + +### Example of {module}.Dockerfile + +```Dockerfile +# Image name: test-run/{module} +FROM test-run/base:latest + +ARG MODULE_NAME={module} +ARG MODULE_DIR=modules/network/$MODULE_NAME + +# Install network service dependencies +# ... + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python + +# Do not specify a CMD or Entrypoint as Test Run will automatically start your service as required +``` + +### Example of start_network_service script + +```bash +#!/bin/bash + +CONFIG_FILE=/etc/network_service/config.conf +# ... + +echo "Starting Network Service..." + +# Perform any required setup steps +# ... + +# Start the network service +# ... + +# Monitor for changes in the config file +# ... + +# Restart the network service when the config changes +# ... +``` + + + + diff --git a/docs/network/addresses.md b/docs/network/addresses.md new file mode 100644 index 000000000..ecaacfd36 --- /dev/null +++ b/docs/network/addresses.md @@ -0,0 +1,18 @@ +# Network Addresses + +Each network service is configured with an IPv4 and IPv6 address. For IPv4 addressing, the last number in the IPv4 address is fixed (ensuring the IP is unique). See below for a table of network addresses: + +| Name | Mac address | IPv4 address | IPv6 address | +|---------------------|----------------------|--------------|------------------------------| +| Internet gateway | 9a:02:57:1e:8f:01 | 10.10.10.1 | fd10:77be:4186::1 | +| DHCP primary | 9a:02:57:1e:8f:02 | 10.10.10.2 | fd10:77be:4186::2 | +| DHCP secondary | 9a:02:57:1e:8f:03 | 10.10.10.3 | fd10:77be:4186::3 | +| DNS server | 9a:02:57:1e:8f:04 | 10.10.10.4 | fd10:77be:4186::4 | +| NTP server | 9a:02:57:1e:8f:05 | 10.10.10.5 | fd10:77be:4186::5 | +| Radius authenticator| 9a:02:57:1e:8f:07 | 10.10.10.7 | fd10:77be:4186::7 | +| Active test module | 9a:02:57:1e:8f:09 | 10.10.10.9 | fd10:77be:4186::9 | + + +The default network range is 10.10.10.0/24 and devices will be assigned addresses in that range via DHCP. The range may change when requested by a test module. In which case, network services will be restarted and accessible on the new range, with the same final host ID. The default IPv6 network is fd10:77be:4186::/64 and addresses will be assigned to devices on the network using IPv6 SLAAC. + +When creating a new network module, please ensure that the ip_index value in the module_config.json is unique otherwise unexpected behaviour will occur. \ No newline at end of file diff --git a/framework/python/src/net_orc/network_validator.py b/framework/python/src/net_orc/network_validator.py index a4c51eb2d..f82787af5 100644 --- a/framework/python/src/net_orc/network_validator.py +++ b/framework/python/src/net_orc/network_validator.py @@ -193,7 +193,7 @@ def _get_os_user(self): LOGGER.error('An OS error occurred while retrieving the login name.') except Exception as error: # Catch any other unexpected exceptions - LOGGER.error('An exception occurred:', error) + LOGGER.error('An exception occurred:', error) return user def _get_user(self): @@ -203,15 +203,15 @@ def _get_user(self): except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: # Handle specific exceptions individually if isinstance(e, KeyError): - LOGGER.error("USER environment variable not set or unavailable.") + LOGGER.error('USER environment variable not set or unavailable.') elif isinstance(e, ImportError): - LOGGER.error("Unable to import the getpass module.") + LOGGER.error('Unable to import the getpass module.') elif isinstance(e, ModuleNotFoundError): - LOGGER.error("The getpass module was not found.") + LOGGER.error('The getpass module was not found.') elif isinstance(e, OSError): - LOGGER.error("An OS error occurred while retrieving the username.") + LOGGER.error('An OS error occurred while retrieving the username.') else: - LOGGER.error("An exception occurred:", e) + LOGGER.error('An exception occurred:', e) return user def _get_device_status(self, module): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 037dbb650..fef4e5bb5 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -13,16 +13,14 @@ # limitations under the License. """Provides high level management of the test orchestrator.""" -import getpass import os import json import time import shutil import docker from docker.types import Mount -from common import logger +from common import logger, util from test_orc.module import TestModule -from common import util LOG_NAME = "test_orc" LOGGER = logger.get_logger("test_orc") @@ -61,7 +59,7 @@ def start(self): # Setup the output directory self._host_user = util.get_host_user() os.makedirs(RUNTIME_DIR, exist_ok=True) - util.run_command(f'chown -R {self._host_user} {RUNTIME_DIR}') + util.run_command(f"chown -R {self._host_user} {RUNTIME_DIR}") self._load_test_modules() self.build_test_modules() @@ -102,7 +100,7 @@ def _generate_results(self, device): results[module.name] = module_results except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: - LOGGER.error("Error occured whilst obbtaining results for module " + module.name) + LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") LOGGER.debug(results_error) out_file = os.path.join( @@ -140,18 +138,19 @@ def _run_test_module(self, module, device): container_runtime_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/" + module.name) - network_runtime_dir = os.path.join(self._root_path, "runtime/network") os.makedirs(container_runtime_dir) + network_runtime_dir = os.path.join(self._root_path, "runtime/network") + device_startup_capture = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/startup.pcap") - util.run_command(f'chown -R {self._host_user} {device_startup_capture}') + util.run_command(f"chown -R {self._host_user} {device_startup_capture}") device_monitor_capture = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/monitor.pcap") - util.run_command(f'chown -R {self._host_user} {device_monitor_capture}') + util.run_command(f"chown -R {self._host_user} {device_monitor_capture}") client = docker.from_env() diff --git a/local/system.json.example b/local/system.json.example index ecf480104..e99e013f3 100644 --- a/local/system.json.example +++ b/local/system.json.example @@ -1,10 +1,10 @@ -{ - "network": { - "device_intf": "enx123456789123", - "internet_intf": "enx123456789124" - }, - "log_level": "INFO", - "startup_timeout": 60, - "monitor_period": 300, - "runtime": 1200 +{ + "network": { + "device_intf": "enx123456789123", + "internet_intf": "enx123456789124" + }, + "log_level": "INFO", + "startup_timeout": 60, + "monitor_period": 300, + "runtime": 1200 } \ No newline at end of file diff --git a/modules/network/base/base.Dockerfile b/modules/network/base/base.Dockerfile index f8fa43c57..ac964a99d 100644 --- a/modules/network/base/base.Dockerfile +++ b/modules/network/base/base.Dockerfile @@ -17,10 +17,14 @@ FROM ubuntu:jammy ARG MODULE_NAME=base ARG MODULE_DIR=modules/network/$MODULE_NAME +ARG COMMON_DIR=framework/python/src/common # Install common software RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix +# Install common python modules +COPY $COMMON_DIR/ /testrun/python/src/common + # Setup the base python requirements COPY $MODULE_DIR/python /testrun/python diff --git a/modules/network/base/bin/setup_python_path b/modules/network/base/bin/setup_python_path new file mode 100644 index 000000000..3e30e965d --- /dev/null +++ b/modules/network/base/bin/setup_python_path @@ -0,0 +1,25 @@ +#!/bin/bash + +ROOT_DIRECTORY="/testrun/python/src" + +# Function to recursively add subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath() { + local directory=$1 + local subdirectories=( "$directory"/* ) + local subdirectory + + for subdirectory in "${subdirectories[@]}"; do + if [[ -d "$subdirectory" && ! "$subdirectory" = *'__pycache__' ]]; then + export PYTHONPATH="$PYTHONPATH:$subdirectory" + add_subdirectories_to_pythonpath "$subdirectory" + fi + done +} + +# Set PYTHONPATH initially to an empty string +export PYTHONPATH="$ROOT_DIRECTORY" + +# Add all subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath "$ROOT_DIRECTORY" + +echo "$PYTHONPATH" \ No newline at end of file diff --git a/modules/network/base/bin/start_grpc b/modules/network/base/bin/start_grpc index 56f915db7..840bea65f 100644 --- a/modules/network/base/bin/start_grpc +++ b/modules/network/base/bin/start_grpc @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -GRPC_DIR="/testrun/python/src/grpc" +GRPC_DIR="/testrun/python/src/grpc_server" GRPC_PROTO_DIR="proto" GRPC_PROTO_FILE="grpc.proto" @@ -22,10 +22,10 @@ GRPC_PROTO_FILE="grpc.proto" pushd $GRPC_DIR >/dev/null 2>&1 #Build the grpc proto file every time before starting server -python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. +python3 -u -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. popd >/dev/null 2>&1 #Start the grpc server -python3 -u $GRPC_DIR/start_server.py $@ +python3 -u $GRPC_DIR/start_server.py $@ & diff --git a/modules/network/base/bin/start_module b/modules/network/base/bin/start_module index e00747b43..8e8cb5e4b 100644 --- a/modules/network/base/bin/start_module +++ b/modules/network/base/bin/start_module @@ -29,7 +29,7 @@ useradd $HOST_USER sysctl net.ipv6.conf.all.disable_ipv6=0 sysctl -p -#Read in the config file +# Read in the config file CONF_FILE="/testrun/conf/module_config.json" CONF=`cat $CONF_FILE` @@ -60,10 +60,16 @@ else INTF=$DEFINED_IFACE fi -echo "Starting module $MODULE_NAME on local interface $INTF..." +# Setup the PYTHONPATH so all imports work as expected +echo "Setting up PYTHONPATH..." +export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" +echo "Configuring binary files..." $BIN_DIR/setup_binaries $BIN_DIR +echo "Starting module $MODULE_NAME on local interface $INTF..." + # Wait for interface to become ready $BIN_DIR/wait_for_interface $INTF @@ -80,14 +86,14 @@ then if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] then echo "gRPC port resolved from config: $GRPC_PORT" - $BIN_DIR/start_grpc "-p $GRPC_PORT" & + $BIN_DIR/start_grpc "-p $GRPC_PORT" else - $BIN_DIR/start_grpc & + $BIN_DIR/start_grpc fi fi -#Small pause to let all core services stabalize +# Small pause to let all core services stabalize sleep 3 -#Start the networking service +# Start the networking service $BIN_DIR/start_network_service $MODULE_NAME $INTF \ No newline at end of file diff --git a/modules/network/base/python/requirements.txt b/modules/network/base/python/requirements.txt index 9c4e2b056..9d9473d74 100644 --- a/modules/network/base/python/requirements.txt +++ b/modules/network/base/python/requirements.txt @@ -1,2 +1,3 @@ grpcio -grpcio-tools \ No newline at end of file +grpcio-tools +netifaces \ No newline at end of file diff --git a/modules/network/base/python/src/grpc/start_server.py b/modules/network/base/python/src/grpc_server/start_server.py similarity index 100% rename from modules/network/base/python/src/grpc/start_server.py rename to modules/network/base/python/src/grpc_server/start_server.py diff --git a/modules/network/base/python/src/logger.py b/modules/network/base/python/src/logger.py index 8893b1e8d..998a4aaae 100644 --- a/modules/network/base/python/src/logger.py +++ b/modules/network/base/python/src/logger.py @@ -35,7 +35,7 @@ log_level = logging.getLevelName(log_level_str) except OSError: # TODO: Print out warning that log level is incorrect or missing - LOG_LEVEL = _DEFAULT_LEVEL + log_level = _DEFAULT_LEVEL log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) diff --git a/modules/network/dhcp-1/bin/isc-dhcp-service b/modules/network/dhcp-1/bin/isc-dhcp-service new file mode 100644 index 000000000..de029515b --- /dev/null +++ b/modules/network/dhcp-1/bin/isc-dhcp-service @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CONFIG_FILE=/etc/dhcp/dhcpd.conf +DHCP_PID_FILE=/var/run/dhcpd.pid +DHCP_LOG_FILE=/runtime/network/dhcp1-dhcpd.log + +stop_dhcp(){ + # Directly kill by PID file reference + if [ -f "$DHCP_PID_FILE" ]; then + kill -9 $(cat $DHCP_PID_FILE) || true + rm -f $DHCP_PID_FILE + fi +} + +start_dhcp(){ + /usr/sbin/dhcpd -d &> $DHCP_LOG_FILE & +} + +case "$1" in + start) + start_dhcp + ;; + stop) + stop_dhcp + ;; + restart) + stop_dhcp + sleep 1 + start_dhcp + ;; + status) + if [ -f "$DHCP_PID_FILE" ]; then + echo "isc-dhcp service is running." + else + echo "isc-dhcp service is not running." + fi + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac \ No newline at end of file diff --git a/modules/network/dhcp-1/bin/radvd-service b/modules/network/dhcp-1/bin/radvd-service new file mode 100644 index 000000000..1cfe499cb --- /dev/null +++ b/modules/network/dhcp-1/bin/radvd-service @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RA_PID_FILE=/var/run/radvd/radvd.pid +RA_LOG_FILE=/runtime/network/dhcp1-radvd.log + +stop_radvd(){ + # Directly kill by PID file reference + if [ -f "$RA_PID_FILE" ]; then + kill -9 $(cat $RA_PID_FILE) || true + rm -f $RA_PID_FILE + fi +} + +start_radvd(){ + /usr/sbin/radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE +} + +case "$1" in + start) + start_radvd + ;; + stop) + stop_radvd + ;; + restart) + stop_radvd + sleep 1 + start_radvd + ;; + status) + if [ -f "$RA_PID_FILE" ]; then + echo "radvd service is running." + else + echo "radvd service is not running." + fi + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac \ No newline at end of file diff --git a/modules/network/dhcp-1/bin/start_network_service b/modules/network/dhcp-1/bin/start_network_service index fbeede871..945313dd3 100644 --- a/modules/network/dhcp-1/bin/start_network_service +++ b/modules/network/dhcp-1/bin/start_network_service @@ -20,7 +20,7 @@ DHCP_LOG_FILE=/runtime/network/dhcp1-dhcpd.log RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp1-radvd.log -echo "Starrting Network Service..." +echo "Starting Network Service..." #Enable IPv6 Forwarding sysctl net.ipv6.conf.all.forwarding=1 @@ -29,63 +29,23 @@ sysctl -p # Create leases file if needed touch /var/lib/dhcp/dhcpd.leases -#Create directory for radvd +# Create directory for radvd mkdir /var/run/radvd -#Create and set permissions on the log files +# Create and set permissions on the log files touch $DHCP_LOG_FILE touch $RA_LOG_FILE chown $HOST_USER $DHCP_LOG_FILE chown $HOST_USER $RA_LOG_FILE - -#Move the config files to the correct location +# Move the config files to the correct location +cp /testrun/conf/isc-dhcp-server /etc/default/ cp /testrun/conf/dhcpd.conf /etc/dhcp/dhcpd.conf cp /testrun/conf/radvd.conf /etc/radvd.conf -# Restart dhcp server when config changes -while true; do - - new_checksum=$(md5sum $CONFIG_FILE) - - if [ "$checksum" == "$new_checksum" ]; then - sleep 2 - continue - fi - - echo Config changed. Restarting dhcp server at $(date).. - - if [ -f $DHCP_PID_FILE ]; then - kill -9 $(cat $DHCP_PID_FILE) || true - rm -f $DHCP_PID_FILE - fi - - if [ -f $RA_PID_FILE ]; then - kill -9 $(cat $RA_PID_FILE) || true - rm -f $RA_PID_FILE - fi - - checksum=$new_checksum - - echo Starting isc-dhcp-server at $(date) - - radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE - dhcpd -d &> $DHCP_LOG_FILE & - - while [ ! -f $DHCP_PID_FILE ]; do - echo Waiting for $DHCP_PID_FILE... - sleep 2 - done - - echo $DHCP_PID_FILE now available - - while [ ! -f $RA_PID_FILE ]; do - echo Waiting for $RA_PID_FILE... - sleep 2 - done - - echo $RA_PID_FILE now available - - echo Server now stable +# Move the service files to the correct location +cp /testrun/bin/isc-dhcp-service /usr/local/bin/ +cp /testrun/bin/radvd-service /usr/local/bin/ -done \ No newline at end of file +# Start the DHCP Server +python3 -u /testrun/python/src/grpc_server/dhcp_server.py \ No newline at end of file diff --git a/modules/network/dhcp-1/conf/dhcpd.conf b/modules/network/dhcp-1/conf/dhcpd.conf index 9f4fe1c28..39f67c3b8 100644 --- a/modules/network/dhcp-1/conf/dhcpd.conf +++ b/modules/network/dhcp-1/conf/dhcpd.conf @@ -1,16 +1,17 @@ -default-lease-time 300; +default-lease-time 30; +max-lease-time 30; failover peer "failover-peer" { - primary; - address 10.10.10.2; - port 847; - peer address 10.10.10.3; - peer port 647; - max-response-delay 60; - max-unacked-updates 10; - mclt 3600; - split 128; - load balance max seconds 3; + primary; + address 10.10.10.2; + port 847; + peer address 10.10.10.3; + peer port 647; + max-response-delay 60; + max-unacked-updates 10; + mclt 30; + split 128; + load balance max seconds 3; } subnet 10.10.10.0 netmask 255.255.255.0 { @@ -19,8 +20,10 @@ subnet 10.10.10.0 netmask 255.255.255.0 { option broadcast-address 10.10.10.255; option routers 10.10.10.1; option domain-name-servers 10.10.10.4; + interface veth0; + authoritative; pool { - failover peer "failover-peer"; - range 10.10.10.10 10.10.10.20; + failover peer "failover-peer"; + range 10.10.10.10 10.10.10.20; } -} +} \ No newline at end of file diff --git a/modules/network/dhcp-1/conf/isc-dhcp-server b/modules/network/dhcp-1/conf/isc-dhcp-server new file mode 100644 index 000000000..4a4aa09f9 --- /dev/null +++ b/modules/network/dhcp-1/conf/isc-dhcp-server @@ -0,0 +1,4 @@ +# On what interfaces should the DHCP server (dhcpd) serve DHCP requests? +# Separate multiple interfaces with spaces, e.g. "eth0 eth1". +INTERFACESv4="veth0" +#INTERFACESv6="veth0" diff --git a/modules/network/dhcp-1/conf/module_config.json b/modules/network/dhcp-1/conf/module_config.json index 4a41eee3f..cf1f59a1e 100644 --- a/modules/network/dhcp-1/conf/module_config.json +++ b/modules/network/dhcp-1/conf/module_config.json @@ -1,26 +1,26 @@ -{ - "config": { - "meta": { - "name": "dhcp-1", - "display_name": "DHCP Primary", - "description": "Primary DHCP server with IPv6 SLAAC" - }, - "network": { - "interface": "veth0", - "enable_wan": false, - "ip_index": 2 - }, - "grpc":{ - "port": 5001 - }, - "docker": { - "depends_on": "base", - "mounts": [ - { - "source": "runtime/network", - "target": "/runtime/network" - } - ] - } - } +{ + "config": { + "meta": { + "name": "dhcp-1", + "display_name": "DHCP Primary", + "description": "Primary DHCP server with IPv6 SLAAC" + }, + "network": { + "interface": "veth0", + "enable_wan": false, + "ip_index": 2 + }, + "grpc":{ + "port": 5001 + }, + "docker": { + "depends_on": "base", + "mounts": [ + { + "source": "runtime/network", + "target": "/runtime/network" + } + ] + } + } } \ No newline at end of file diff --git a/modules/network/dhcp-1/conf/radvd.conf b/modules/network/dhcp-1/conf/radvd.conf index f6d6f30d9..0cc500fd5 100644 --- a/modules/network/dhcp-1/conf/radvd.conf +++ b/modules/network/dhcp-1/conf/radvd.conf @@ -7,6 +7,6 @@ interface veth0 prefix fd10:77be:4186::/64 { AdvOnLink on; AdvAutonomous on; - AdvRouterAddr on; + AdvRouterAddr on; }; -}; \ No newline at end of file +}; diff --git a/modules/network/dhcp-1/dhcp-1.Dockerfile b/modules/network/dhcp-1/dhcp-1.Dockerfile index a4eb8d90a..6b941d878 100644 --- a/modules/network/dhcp-1/dhcp-1.Dockerfile +++ b/modules/network/dhcp-1/dhcp-1.Dockerfile @@ -18,8 +18,14 @@ FROM test-run/base:latest ARG MODULE_NAME=dhcp-1 ARG MODULE_DIR=modules/network/$MODULE_NAME +# Install all necessary packages +RUN apt-get install -y wget + +#Update the oui.txt file from ieee +RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ + # Install dhcp server -RUN apt-get install -y isc-dhcp-server radvd +RUN apt-get install -y isc-dhcp-server radvd systemd # Copy over all configuration files COPY $MODULE_DIR/conf /testrun/conf @@ -28,4 +34,4 @@ COPY $MODULE_DIR/conf /testrun/conf COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY $MODULE_DIR/python /testrun/python +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py deleted file mode 100644 index 99d6bdebd..000000000 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains all the necessary classes to maintain the -DHCP server's configuration""" -import re - -CONFIG_FILE = '/etc/dhcp/dhcpd.conf' -CONFIG_FILE_TEST = 'network/modules/dhcp-1/conf/dhcpd.conf' - -DEFAULT_LEASE_TIME_KEY = 'default-lease-time' - - -class DHCPConfig: - """Represents the DHCP Servers configuration and gives access to modify it""" - - def __init__(self): - self._default_lease_time = 300 - self.subnets = [] - self._peer = None - - def write_config(self): - conf = str(self) - print('Writing config: \n' + conf) - with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: - conf_file.write(conf) - - def resolve_config(self): - with open(CONFIG_FILE, 'r', encoding='UTF-8') as f: - conf = f.read() - self.resolve_subnets(conf) - self._peer = DHCPFailoverPeer(conf) - - def resolve_subnets(self, conf): - self.subnets = [] - regex = r'(subnet.*)' - subnets = re.findall(regex, conf, re.MULTILINE | re.DOTALL) - for subnet in subnets: - dhcp_subnet = DHCPSubnet(subnet) - self.subnets.append(dhcp_subnet) - - def set_range(self, start, end, subnet=0, pool=0): - print('Setting Range for pool ') - print(self.subnets[subnet].pools[pool]) - self.subnets[subnet].pools[pool].range_start = start - self.subnets[subnet].pools[pool].range_end = end - - # def resolve_settings(self, conf): - # lines = conf.split('\n') - # for line in lines: - # if DEFAULT_LEASE_TIME_KEY in line: - # self._default_lease_time = line.strip().split( - # DEFAULT_LEASE_TIME_KEY)[1].strip().split(';')[0] - - # self.peer = peer - - def __str__(self): - - config = """\r{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" - - config = config.format(length='multi-line', - DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, - DEFAULT_LEASE_TIME=self._default_lease_time) - - config += '\n\n' + str(self.peer) - for subnet in self._subnets: - config += '\n\n' + str(subnet) - return str(config) - - -FAILOVER_PEER_KEY = 'failover peer' -PRIMARY_KEY = 'primary' -ADDRESS_KEY = 'address' -PORT_KEY = 'port' -PEER_ADDRESS_KEY = 'peer address' -PEER_PORT_KEY = 'peer port' -MAX_RESPONSE_DELAY_KEY = 'max-response-delay' -MAX_UNACKED_UPDATES_KEY = 'max-unacked-updates' -MCLT_KEY = 'mclt' -SPLIT_KEY = 'split' -LOAD_BALANCE_MAX_SECONDS_KEY = 'load balance max seconds' - - -class DHCPFailoverPeer: - """Contains all information to define the DHCP failover peer""" - - def __init__(self, config): - self.name = None - self.primary = False - self.address = None - self.port = None - self.peer_address = None - self.peer_port = None - self.max_response_delay = None - self.max_unacked_updates = None - self.mclt = None - self.split = None - self.load_balance_max_seconds = None - self.peer = None - - self.resolve_peer(config) - - def __str__(self): - config = '{FAILOVER_PEER_KEY} \"{FAILOVER_PEER}\" {{\n' - config += '\tprimary;' if self.primary else 'secondary;' - config += """\n\t{ADDRESS_KEY} {ADDRESS}; - {PORT_KEY} {PORT}; - {PEER_ADDRESS_KEY} {PEER_ADDRESS}; - {PEER_PORT_KEY} {PEER_PORT}; - {MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY}; - {MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES}; - {MCLT_KEY} {MCLT}; - {SPLIT_KEY} {SPLIT}; - {LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS}; - \r}}""" - - return config.format( - length='multi-line', - FAILOVER_PEER_KEY=FAILOVER_PEER_KEY, - FAILOVER_PEER=self.name, - ADDRESS_KEY=ADDRESS_KEY, - ADDRESS=self.address, - PORT_KEY=PORT_KEY, - PORT=self.port, - PEER_ADDRESS_KEY=PEER_ADDRESS_KEY, - PEER_ADDRESS=self.peer_address, - PEER_PORT_KEY=PEER_PORT_KEY, - PEER_PORT=self.peer_port, - MAX_RESPONSE_DELAY_KEY=MAX_RESPONSE_DELAY_KEY, - MAX_RESPONSE_DELAY=self.max_response_delay, - MAX_UNACKED_UPDATES_KEY=MAX_UNACKED_UPDATES_KEY, - MAX_UNACKED_UPDATES=self.max_unacked_updates, - MCLT_KEY=MCLT_KEY, - MCLT=self.mclt, - SPLIT_KEY=SPLIT_KEY, - SPLIT=self.split, - LOAD_BALANCE_MAX_SECONDS_KEY=LOAD_BALANCE_MAX_SECONDS_KEY, - LOAD_BALANCE_MAX_SECONDS=self.load_balance_max_seconds) - - def resolve_peer(self, conf): - peer = '' - lines = conf.split('\n') - for line in lines: - if line.startswith(FAILOVER_PEER_KEY) or len(peer) > 0: - if len(peer) <= 0: - self.name = line.strip().split(FAILOVER_PEER_KEY)[1].strip().split( - '{')[0].split('\"')[1] - peer += line + '\n' - if PRIMARY_KEY in line: - self.primary = True - elif ADDRESS_KEY in line and PEER_ADDRESS_KEY not in line: - self.address = line.strip().split(ADDRESS_KEY)[1].strip().split( - ';')[0] - elif PORT_KEY in line and PEER_PORT_KEY not in line: - self.port = line.strip().split(PORT_KEY)[1].strip().split(';')[0] - elif PEER_ADDRESS_KEY in line: - self.peer_address = line.strip().split( - PEER_ADDRESS_KEY)[1].strip().split(';')[0] - elif PEER_PORT_KEY in line: - self.peer_port = line.strip().split(PEER_PORT_KEY)[1].strip().split( - ';')[0] - elif MAX_RESPONSE_DELAY_KEY in line: - self.max_response_delay = line.strip().split( - MAX_RESPONSE_DELAY_KEY)[1].strip().split(';')[0] - elif MAX_UNACKED_UPDATES_KEY in line: - self.max_unacked_updates = line.strip().split( - MAX_UNACKED_UPDATES_KEY)[1].strip().split(';')[0] - elif MCLT_KEY in line: - self.mclt = line.strip().split(MCLT_KEY)[1].strip().split(';')[0] - elif SPLIT_KEY in line: - self.split = line.strip().split(SPLIT_KEY)[1].strip().split(';')[0] - elif LOAD_BALANCE_MAX_SECONDS_KEY in line: - self.load_balance_max_seconds = line.strip().split( - LOAD_BALANCE_MAX_SECONDS_KEY)[1].strip().split(';')[0] - if line.endswith('}') and len(peer) > 0: - break - self.peer = peer - - -NTP_OPTION_KEY = 'option ntp-servers' -SUBNET_MASK_OPTION_KEY = 'option subnet-mask' -BROADCAST_OPTION_KEY = 'option broadcast-address' -ROUTER_OPTION_KEY = 'option routers' -DNS_OPTION_KEY = 'option domain-name-servers' - - -class DHCPSubnet: - """Represents the DHCP Servers subnet configuration""" - - def __init__(self, subnet): - self._ntp_servers = None - self._subnet_mask = None - self._broadcast = None - self._routers = None - self._dns_servers = None - self.pools = [] - - self.resolve_subnet(subnet) - self.resolve_pools(subnet) - - def __str__(self): - config = """subnet 10.10.10.0 netmask {SUBNET_MASK_OPTION} {{ - \r\t{NTP_OPTION_KEY} {NTP_OPTION}; - \r\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION}; - \r\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION}; - \r\t{ROUTER_OPTION_KEY} {ROUTER_OPTION}; - \r\t{DNS_OPTION_KEY} {DNS_OPTION};""" - - config = config.format(length='multi-line', - NTP_OPTION_KEY=NTP_OPTION_KEY, - NTP_OPTION=self._ntp_servers, - SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, - SUBNET_MASK_OPTION=self._subnet_mask, - BROADCAST_OPTION_KEY=BROADCAST_OPTION_KEY, - BROADCAST_OPTION=self._broadcast, - ROUTER_OPTION_KEY=ROUTER_OPTION_KEY, - ROUTER_OPTION=self._routers, - DNS_OPTION_KEY=DNS_OPTION_KEY, - DNS_OPTION=self._dns_servers) - for pool in self.pools: - config += '\n\t' + str(pool) - - config += '\n\r}' - return config - - def resolve_subnet(self, subnet): - subnet_parts = subnet.split('\n') - for part in subnet_parts: - if NTP_OPTION_KEY in part: - self._ntp_servers = part.strip().split(NTP_OPTION_KEY)[1].strip().split( - ';')[0] - elif SUBNET_MASK_OPTION_KEY in part: - self._subnet_mask = part.strip().split( - SUBNET_MASK_OPTION_KEY)[1].strip().split(';')[0] - elif BROADCAST_OPTION_KEY in part: - self._broadcast = part.strip().split( - BROADCAST_OPTION_KEY)[1].strip().split(';')[0] - elif ROUTER_OPTION_KEY in part: - self._routers = part.strip().split(ROUTER_OPTION_KEY)[1].strip().split( - ';')[0] - elif DNS_OPTION_KEY in part: - self._dns_servers = part.strip().split(DNS_OPTION_KEY)[1].strip().split( - ';')[0] - - def resolve_pools(self, subnet): - regex = r'(pool.*)\}' - pools = re.findall(regex, subnet, re.MULTILINE | re.DOTALL) - for pool in pools: - dhcp_pool = DHCPPool(pool) - self.pools.append(dhcp_pool) - - -FAILOVER_KEY = 'failover peer' -RANGE_KEY = 'range' - - -class DHCPPool: - """Represents a DHCP Servers subnet pool configuration""" - - def __init__(self, pool): - self.failover_peer = None - self.range_start = None - self.range_end = None - self.resolve_pool(pool) - - def __str__(self): - - config = """pool {{ - \r\t\t{FAILOVER_KEY} "{FAILOVER}"; - \r\t\t{RANGE_KEY} {RANGE_START} {RANGE_END}; - \r\t}}""" - - return config.format( - length='multi-line', - FAILOVER_KEY=FAILOVER_KEY, - FAILOVER=self.failover_peer, - RANGE_KEY=RANGE_KEY, - RANGE_START=self.range_start, - RANGE_END=self.range_end, - ) - - def resolve_pool(self, pool): - pool_parts = pool.split('\n') - # pool_parts = pool.split("\n") - for part in pool_parts: - if FAILOVER_KEY in part: - self.failover_peer = part.strip().split(FAILOVER_KEY)[1].strip().split( - ';')[0].replace('\"', '') - if RANGE_KEY in part: - pool_range = part.strip().split(RANGE_KEY)[1].strip().split(';')[0] - self.range_start = pool_range.split(' ')[0].strip() - self.range_end = pool_range.split(' ')[1].strip() diff --git a/modules/network/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc/network_service.py deleted file mode 100644 index 64aab8a07..000000000 --- a/modules/network/dhcp-1/python/src/grpc/network_service.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""gRPC Network Service for the DHCP Server network module""" -import proto.grpc_pb2_grpc as pb2_grpc -import proto.grpc_pb2 as pb2 - -from dhcp_config import DHCPConfig - - -class NetworkService(pb2_grpc.NetworkModule): - """gRPC endpoints for the DHCP Server""" - - def __init__(self): - self._dhcp_config = DHCPConfig() - - def GetDHCPRange(self, request, context): # pylint: disable=W0613 - """ - Resolve the current DHCP configuration and return - the first range from the first subnet in the file - """ - self._dhcp_config.resolve_config() - pool = self._dhcp_config.subnets[0].pools[0] - return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) - - def SetDHCPRange(self, request, context): # pylint: disable=W0613 - """ - Change DHCP configuration and set the - the first range from the first subnet in the configuration - """ - - print('Setting DHCPRange') - print('Start: ' + request.start) - print('End: ' + request.end) - self._dhcp_config.resolve_config() - self._dhcp_config.set_range(request.start, request.end, 0, 0) - self._dhcp_config.write_config() - return pb2.Response(code=200, message='DHCP Range Set') - - def GetStatus(self, request, context): # pylint: disable=W0613 - """ - Return the current status of the network module - """ - # ToDo: Figure out how to resolve the current DHCP status - dhcp_status = True - message = str({'dhcpStatus': dhcp_status}) - return pb2.Response(code=200, message=message) diff --git a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto deleted file mode 100644 index 8e2732620..000000000 --- a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto +++ /dev/null @@ -1,36 +0,0 @@ -syntax = "proto3"; - -service NetworkModule { - - rpc GetDHCPRange(GetDHCPRangeRequest) returns (DHCPRange) {}; - - rpc SetDHCPRange(DHCPRange) returns (Response) {}; - - rpc GetStatus(GetStatusRequest) returns (Response) {}; - - rpc GetIPAddress(GetIPAddressRequest) returns (Response) {}; - - rpc SetLeaseAddress(SetLeaseAddressRequest) returns (Response) {}; - -} - -message Response { - int32 code = 1; - string message = 2; -} - -message DHCPRange { - int32 code = 1; - string start = 2; - string end = 3; -} - -message GetDHCPRangeRequest {} - -message GetIPAddressRequest {} - -message GetStatusRequest {} - -message SetLeaseAddressRequest { - string ipAddress = 1; -} \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/__init__.py b/modules/network/dhcp-1/python/src/grpc_server/__init__.py similarity index 100% rename from modules/network/dhcp-1/python/src/grpc/__init__.py rename to modules/network/dhcp-1/python/src/grpc_server/__init__.py diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py new file mode 100644 index 000000000..877d49610 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py @@ -0,0 +1,551 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server's configuration""" +import re +from common import logger + +LOG_NAME = 'dhcp_config' +LOGGER = None +CONFIG_FILE = '/etc/dhcp/dhcpd.conf' +DEFAULT_LEASE_TIME_KEY = 'default-lease-time' +MAX_LEASE_TIME_KEY = 'max-lease-time' + + +class DHCPConfig: + """Represents the DHCP Servers configuration and gives access to modify it""" + + def __init__(self): + self._default_lease_time = 30 + self._max_lease_time = 30 + self._subnets = [] + self._peer = None + self._reserved_hosts = [] + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def add_reserved_host(self, hostname, hw_addr, ip_addr): + host = DHCPReservedHost(hostname=hostname, + hw_addr=hw_addr, + fixed_addr=ip_addr) + self._reserved_hosts.append(host) + + def delete_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host.hw_addr: + self._reserved_hosts.remove(host) + + def disable_failover(self): + self._peer.disable() + for subnet in self._subnets: + subnet.disable_peer() + + def enable_failover(self): + self._peer.enable() + for subnet in self._subnets: + subnet.enable_peer() + + def get_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host.hw_addr: + return host + + def write_config(self, config=None): + if config is None: + conf = str(self) + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(conf) + else: + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(config) + + def _get_config(self, config_file=CONFIG_FILE): + content = None + with open(config_file, 'r', encoding='UTF-8') as f: + content = f.read() + return content + + def make(self, conf): + try: + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: # pylint: disable=W0718 + print('Failed to make DHCPConfig: ' + str(e)) + + def resolve_config(self, config_file=CONFIG_FILE): + try: + conf = self._get_config(config_file) + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: # pylint: disable=W0718 + print('Failed to resolve config: ' + str(e)) + + def resolve_subnets(self, conf): + subnets = [] + regex = r'(subnet.*)' + subnets_conf = re.findall(regex, conf, re.MULTILINE | re.DOTALL) + for subnet in subnets_conf: + dhcp_subnet = DHCPSubnet(subnet) + subnets.append(dhcp_subnet) + return subnets + + def resolve_reserved_hosts(self, conf): + hosts = [] + host_start = 0 + while True: + host_start = conf.find('host', host_start) + if host_start < 0: + break + else: + host_end = conf.find('}', host_start) + host = DHCPReservedHost(config=conf[host_start:host_end + 1]) + hosts.append(host) + host_start = host_end + 1 + return hosts + + def set_range(self, start, end, subnet=0, pool=0): + # Calculate the subnet from the range + octets = start.split('.') + octets[-1] = '0' + dhcp_subnet = '.'.join(octets) + + # Calcualte the netmask from the range + prefix = self.calculate_prefix_length(start, end) + netmask = self.calculate_netmask(prefix) + + #Update the subnet, range and netmask + self._subnets[subnet].set_subnet(dhcp_subnet, netmask) + self._subnets[subnet].pools[pool].set_range(start, end) + + def calculate_prefix_length(self, start_ip, end_ip): + start_octets = start_ip.split('.') + end_octets = end_ip.split('.') + + start_int = int( + ''.join(format(int(octet), '08b') for octet in start_octets), 2) + end_int = int(''.join(format(int(octet), '08b') for octet in end_octets), 2) + + xor_result = start_int ^ end_int + prefix_length = 32 - xor_result.bit_length() + + return prefix_length + + def calculate_netmask(self, prefix_length): + num_network_bits = prefix_length + num_host_bits = 32 - num_network_bits + + netmask_int = (2**num_network_bits - 1) << num_host_bits + netmask_octets = [(netmask_int >> (i * 8)) & 0xff for i in range(3, -1, -1)] + + return '.'.join(str(octet) for octet in netmask_octets) + + def __str__(self): + + config = ('{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};' + if self._default_lease_time is not None else '') + config += ('\n\r{MAX_LEASE_TIME_KEY} {MAX_LEASE_TIME};' + if self._max_lease_time is not None else '') + + # Encode the top level config options + #config = """{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" + config = config.format(length='multi-line', + DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, + DEFAULT_LEASE_TIME=self._default_lease_time, + MAX_LEASE_TIME_KEY=MAX_LEASE_TIME_KEY, + MAX_LEASE_TIME=self._max_lease_time) + + # Encode the failover peer + config += '\n\n' + str(self._peer) + + # Encode the subnets + for subnet in self._subnets: + config += '\n\n' + str(subnet) + + # Encode the reserved hosts + for host in self._reserved_hosts: + config += '\n' + str(host) + + return str(config) + + +FAILOVER_PEER_KEY = 'failover peer' +PRIMARY_KEY = 'primary' +ADDRESS_KEY = 'address' +PORT_KEY = 'port' +PEER_ADDRESS_KEY = 'peer address' +PEER_PORT_KEY = 'peer port' +MAX_RESPONSE_DELAY_KEY = 'max-response-delay' +MAX_UNACKED_UPDATES_KEY = 'max-unacked-updates' +MCLT_KEY = 'mclt' +SPLIT_KEY = 'split' +LOAD_BALANCE_MAX_SECONDS_KEY = 'load balance max seconds' + + +class DHCPFailoverPeer: + """Contains all information to define the DHCP failover peer""" + + def __init__(self, config): + self.name = None + self.primary = False + self.address = None + self.port = None + self.peer_address = None + self.peer_port = None + self.max_response_delay = None + self.max_unacked_updates = None + self.mclt = None + self.split = None + self.load_balance_max_seconds = None + self.peer = None + self.enabled = True + + self.resolve_peer(config) + + def __str__(self): + config = '{FAILOVER_PEER_KEY} \"{FAILOVER_PEER}\" {{\n' + config += '\tprimary;' if self.primary else 'secondary;' + config += '\n\t{ADDRESS_KEY} {ADDRESS};' if self.address is not None else '' + config += '\n\t{PORT_KEY} {PORT};' if self.port is not None else '' + config += ('\n\t{PEER_ADDRESS_KEY} {PEER_ADDRESS};' + if self.peer_address is not None else '') + config += ('\n\t{PEER_PORT_KEY} {PEER_PORT};' + if self.peer_port is not None else '') + config += ('\n\t{MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY};' + if self.max_response_delay is not None else '') + config += ('\n\t{MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES};' + if self.max_unacked_updates is not None else '') + config += '\n\t{MCLT_KEY} {MCLT};' if self.mclt is not None else '' + config += '\n\t{SPLIT_KEY} {SPLIT};' if self.split is not None else '' + config += ('\n\t{LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS};' + if self.load_balance_max_seconds is not None else '') + config += '\n\r}}' + + config = config.format( + length='multi-line', + FAILOVER_PEER_KEY=FAILOVER_PEER_KEY, + FAILOVER_PEER=self.name, + ADDRESS_KEY=ADDRESS_KEY, + ADDRESS=self.address, + PORT_KEY=PORT_KEY, + PORT=self.port, + PEER_ADDRESS_KEY=PEER_ADDRESS_KEY, + PEER_ADDRESS=self.peer_address, + PEER_PORT_KEY=PEER_PORT_KEY, + PEER_PORT=self.peer_port, + MAX_RESPONSE_DELAY_KEY=MAX_RESPONSE_DELAY_KEY, + MAX_RESPONSE_DELAY=self.max_response_delay, + MAX_UNACKED_UPDATES_KEY=MAX_UNACKED_UPDATES_KEY, + MAX_UNACKED_UPDATES=self.max_unacked_updates, + MCLT_KEY=MCLT_KEY, + MCLT=self.mclt, + SPLIT_KEY=SPLIT_KEY, + SPLIT=self.split, + LOAD_BALANCE_MAX_SECONDS_KEY=LOAD_BALANCE_MAX_SECONDS_KEY, + LOAD_BALANCE_MAX_SECONDS=self.load_balance_max_seconds) + + if not self.enabled: + lines = config.strip().split('\n') + for i in range(len(lines) - 1): + lines[i] = '#' + lines[i] + lines[-1] = '#' + lines[-1].strip() # Handle the last line separately + config = '\n'.join(lines) + + return config + + def disable(self): + self.enabled = False + + def enable(self): + self.enabled = True + + def resolve_peer(self, conf): + peer = '' + lines = conf.split('\n') + for line in lines: + if line.startswith(FAILOVER_PEER_KEY) or len(peer) > 0: + if len(peer) <= 0: + self.name = line.strip().split(FAILOVER_PEER_KEY)[1].strip().split( + '{')[0].split('\"')[1] + peer += line + '\n' + if PRIMARY_KEY in line: + self.primary = True + elif ADDRESS_KEY in line and PEER_ADDRESS_KEY not in line: + self.address = line.strip().split(ADDRESS_KEY)[1].strip().split( + ';')[0] + elif PORT_KEY in line and PEER_PORT_KEY not in line: + self.port = line.strip().split(PORT_KEY)[1].strip().split(';')[0] + elif PEER_ADDRESS_KEY in line: + self.peer_address = line.strip().split( + PEER_ADDRESS_KEY)[1].strip().split(';')[0] + elif PEER_PORT_KEY in line: + self.peer_port = line.strip().split(PEER_PORT_KEY)[1].strip().split( + ';')[0] + elif MAX_RESPONSE_DELAY_KEY in line: + self.max_response_delay = line.strip().split( + MAX_RESPONSE_DELAY_KEY)[1].strip().split(';')[0] + elif MAX_UNACKED_UPDATES_KEY in line: + self.max_unacked_updates = line.strip().split( + MAX_UNACKED_UPDATES_KEY)[1].strip().split(';')[0] + elif MCLT_KEY in line: + self.mclt = line.strip().split(MCLT_KEY)[1].strip().split(';')[0] + elif SPLIT_KEY in line: + self.split = line.strip().split(SPLIT_KEY)[1].strip().split(';')[0] + elif LOAD_BALANCE_MAX_SECONDS_KEY in line: + self.load_balance_max_seconds = line.strip().split( + LOAD_BALANCE_MAX_SECONDS_KEY)[1].strip().split(';')[0] + if line.endswith('}') and len(peer) > 0: + break + self.peer = peer + + +SUBNET_KEY = 'subnet' +NTP_OPTION_KEY = 'option ntp-servers' +SUBNET_MASK_OPTION_KEY = 'option subnet-mask' +BROADCAST_OPTION_KEY = 'option broadcast-address' +ROUTER_OPTION_KEY = 'option routers' +DNS_OPTION_KEY = 'option domain-name-servers' +INTERFACE_KEY = 'interface' +AUTHORITATIVE_KEY = 'authoritative' + + +class DHCPSubnet: + """Represents the DHCP Servers subnet configuration""" + + def __init__(self, subnet): + self._authoritative = False + self._subnet = None + self._ntp_servers = None + self._subnet_mask = None + self._broadcast = None + self._routers = None + self._dns_servers = None + self._interface = None + self.pools = [] + + self.resolve_subnet(subnet) + self.resolve_pools(subnet) + + def __str__(self): + config = 'subnet {SUBNET_OPTION} netmask {SUBNET_MASK_OPTION} {{' + config += ('\n\t{NTP_OPTION_KEY} {NTP_OPTION};' + if self._ntp_servers is not None else '') + config += ('\n\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION};' + if self._subnet_mask is not None else '') + config += ('\n\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION};' + if self._broadcast is not None else '') + config += ('\n\t{ROUTER_OPTION_KEY} {ROUTER_OPTION};' + if self._routers is not None else '') + config += ('\n\t{DNS_OPTION_KEY} {DNS_OPTION};' + if self._dns_servers is not None else '') + config += ('\n\t{INTERFACE_KEY} {INTERFACE_OPTION};' + if self._interface is not None else '') + config += '\n\t{AUTHORITATIVE_KEY};' if self._authoritative else '' + + config = config.format(length='multi-line', + SUBNET_OPTION=self._subnet, + NTP_OPTION_KEY=NTP_OPTION_KEY, + NTP_OPTION=self._ntp_servers, + SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, + SUBNET_MASK_OPTION=self._subnet_mask, + BROADCAST_OPTION_KEY=BROADCAST_OPTION_KEY, + BROADCAST_OPTION=self._broadcast, + ROUTER_OPTION_KEY=ROUTER_OPTION_KEY, + ROUTER_OPTION=self._routers, + DNS_OPTION_KEY=DNS_OPTION_KEY, + DNS_OPTION=self._dns_servers, + INTERFACE_KEY=INTERFACE_KEY, + INTERFACE_OPTION=self._interface, + AUTHORITATIVE_KEY=AUTHORITATIVE_KEY) + + # if not self._authoritative: + # config = config.replace(AUTHORITATIVE_KEY, '#' + AUTHORITATIVE_KEY) + + for pool in self.pools: + config += '\n\t' + str(pool) + + config += '\n}' + return config + + def disable_peer(self): + for pool in self.pools: + pool.disable_peer() + + def enable_peer(self): + for pool in self.pools: + pool.enable_peer() + + def set_subnet(self, subnet, netmask=None): + if netmask is None: + netmask = '255.255.255.0' + self._subnet = subnet + self._subnet_mask = netmask + + # Calculate the broadcast from the subnet and netmask + broadcast = self.calculate_broadcast_address(subnet, netmask) + self._broadcast = broadcast + + def calculate_broadcast_address(self, subnet_address, netmask): + subnet_octets = subnet_address.split('.') + netmask_octets = netmask.split('.') + + subnet_int = int( + ''.join(format(int(octet), '08b') for octet in subnet_octets), 2) + netmask_int = int( + ''.join(format(int(octet), '08b') for octet in netmask_octets), 2) + + broadcast_int = subnet_int | (~netmask_int & 0xffffffff) + broadcast_octets = [(broadcast_int >> (i * 8)) & 0xff + for i in range(3, -1, -1)] + + return '.'.join(str(octet) for octet in broadcast_octets) + + def resolve_subnet(self, subnet): + subnet_parts = subnet.split('\n') + for part in subnet_parts: + if part.strip().startswith(SUBNET_KEY): + self._subnet = part.strip().split()[1] + elif NTP_OPTION_KEY in part: + self._ntp_servers = part.strip().split(NTP_OPTION_KEY)[1].strip().split( + ';')[0] + elif SUBNET_MASK_OPTION_KEY in part: + self._subnet_mask = part.strip().split( + SUBNET_MASK_OPTION_KEY)[1].strip().split(';')[0] + elif BROADCAST_OPTION_KEY in part: + self._broadcast = part.strip().split( + BROADCAST_OPTION_KEY)[1].strip().split(';')[0] + elif ROUTER_OPTION_KEY in part: + self._routers = part.strip().split(ROUTER_OPTION_KEY)[1].strip().split( + ';')[0] + elif DNS_OPTION_KEY in part: + self._dns_servers = part.strip().split(DNS_OPTION_KEY)[1].strip().split( + ';')[0] + elif INTERFACE_KEY in part: + self._interface = part.strip().split(INTERFACE_KEY)[1].strip().split( + ';')[0] + elif AUTHORITATIVE_KEY in part: + self._authoritative = True + + def resolve_pools(self, subnet): + regex = r'(pool.*)\}' + pools = re.findall(regex, subnet, re.MULTILINE | re.DOTALL) + for pool in pools: + dhcp_pool = DHCPPool(pool) + self.pools.append(dhcp_pool) + + +FAILOVER_KEY = 'failover peer' +RANGE_KEY = 'range' + + +class DHCPPool: + """Represents a DHCP Servers subnet pool configuration""" + + def __init__(self, pool): + self.failover_peer = None + self.range_start = None + self.range_end = None + self.resolve_pool(pool) + self._peer_enabled = True + + def __str__(self): + config = 'pool {{' + config += ('\n\t\t{FAILOVER_KEY} "{FAILOVER}";' + if self.failover_peer is not None else '') + config += ('\n\t\t{RANGE_KEY} {RANGE_START} {RANGE_END};' + if self.range_start is not None and self.range_end is not None + else '') + config += '\n\t}}' + + config = config.format( + length='multi-line', + FAILOVER_KEY=FAILOVER_KEY, + FAILOVER=self.failover_peer, + RANGE_KEY=RANGE_KEY, + RANGE_START=self.range_start, + RANGE_END=self.range_end, + ) + + if not self._peer_enabled: + config = config.replace(FAILOVER_KEY, '#' + FAILOVER_KEY) + + return config + + def disable_peer(self): + self._peer_enabled = False + + def enable_peer(self): + self._peer_enabled = True + + def set_range(self, start, end): + self.range_start = start + self.range_end = end + + def resolve_pool(self, pool): + pool_parts = pool.split('\n') + for part in pool_parts: + if FAILOVER_KEY in part: + self.failover_peer = part.strip().split(FAILOVER_KEY)[1].strip().split( + ';')[0].replace('\"', '') + if RANGE_KEY in part: + pool_range = part.strip().split(RANGE_KEY)[1].strip().split(';')[0] + self.range_start = pool_range.split(' ')[0].strip() + self.range_end = pool_range.split(' ')[1].strip() + + +HOST_KEY = 'host' +HARDWARE_KEY = 'hardware ethernet' +FIXED_ADDRESS_KEY = 'fixed-address' + + +class DHCPReservedHost: + """Represents a DHCP Servers subnet pool configuration""" + + def __init__(self, hostname=None, hw_addr=None, fixed_addr=None, config=None): + if config is None: + self.host = hostname + self.hw_addr = hw_addr + self.fixed_addr = fixed_addr + else: + self.resolve_host(config) + + def __str__(self): + + config = """{HOST_KEY} {HOSTNAME} {{ + \r\t{HARDWARE_KEY} {HW_ADDR}; + \r\t{FIXED_ADDRESS_KEY} {RESERVED_IP}; + \r}}""" + + config = config.format( + length='multi-line', + HOST_KEY=HOST_KEY, + HOSTNAME=self.host, + HARDWARE_KEY=HARDWARE_KEY, + HW_ADDR=self.hw_addr, + FIXED_ADDRESS_KEY=FIXED_ADDRESS_KEY, + RESERVED_IP=self.fixed_addr, + ) + return config + + def resolve_host(self, reserved_host): + host_parts = reserved_host.split('\n') + for part in host_parts: + if HOST_KEY in part: + self.host = part.strip().split(HOST_KEY)[1].strip().split('{')[0] + elif HARDWARE_KEY in part: + self.hw_addr = part.strip().split(HARDWARE_KEY)[1].strip().split(';')[0] + elif FIXED_ADDRESS_KEY in part: + self.fixed_addr = part.strip().split( + FIXED_ADDRESS_KEY)[1].strip().split(';')[0] diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py new file mode 100644 index 000000000..4bc1bd52d --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py @@ -0,0 +1,116 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit Testing for the DHCP Server config""" +import unittest +from dhcp_config import DHCPConfig +import os + +CONFIG_FILE = 'conf/dhcpd.conf' +DHCP_CONFIG = None + + +def get_config_file_path(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + module_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(current_dir)))) + conf_file = os.path.join(module_dir, CONFIG_FILE) + return conf_file + + +def get_config(): + dhcp_config = DHCPConfig() + dhcp_config.resolve_config(get_config_file_path()) + return dhcp_config + + +class DHCPConfigTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + # Resolve the config + global DHCP_CONFIG + DHCP_CONFIG = get_config() + + def test_resolve_config(self): + print('Test Resolve Config:\n' + str(DHCP_CONFIG)) + + # Resolve the raw config file + with open(get_config_file_path(), 'r', encoding='UTF-8') as f: + lines = f.readlines() + + # Get the resolved config as a + conf_parts = str(DHCP_CONFIG).split('\n') + + # dhcpd conf is not picky about spacing so we just + # need to check contents of each line for matching + # to make sure evertying matches + for i in range(len(lines)): + self.assertEqual(lines[i].strip(), conf_parts[i].strip()) + + def test_disable_failover(self): + DHCP_CONFIG.disable_failover() + print('Test Disable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertTrue(line.startswith('#')) + + def test_enable_failover(self): + DHCP_CONFIG.enable_failover() + print('Test Enable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertFalse(line.startswith('#')) + + def test_add_reserved_host(self): + DHCP_CONFIG.add_reserved_host('test', '00:11:22:33:44:55', '192.168.10.5') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print('AddHostConfig:\n' + str(DHCP_CONFIG)) + + def test_delete_reserved_host(self): + DHCP_CONFIG.delete_reserved_host('00:11:22:33:44:55') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNone(host) + print('DeleteHostConfig:\n' + str(DHCP_CONFIG)) + + def test_resolve_config_with_hosts(self): + DHCP_CONFIG.add_reserved_host('test', '00:11:22:33:44:55', '192.168.10.5') + config_with_hosts = DHCPConfig() + config_with_hosts.make(str(DHCP_CONFIG)) + host = config_with_hosts.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print('ResolveConfigWithHosts:\n' + str(config_with_hosts)) + + def test_set_subnet_range(self): + range_start = '10.0.0.100' + range_end = '10.0.0.200' + DHCP_CONFIG.set_range(range_start, range_end) + subnets = DHCP_CONFIG.resolve_subnets(str(DHCP_CONFIG)) + pool = subnets[0].pools[0] + self.assertTrue(pool.range_start == range_start + and pool.range_end == range_end) + print('SetSubnetRange:\n' + str(DHCP_CONFIG)) + +if __name__ == '__main__': + suite = unittest.TestSuite() + suite.addTest(DHCPConfigTest('test_resolve_config')) + suite.addTest(DHCPConfigTest('test_disable_failover')) + suite.addTest(DHCPConfigTest('test_enable_failover')) + suite.addTest(DHCPConfigTest('test_add_reserved_host')) + suite.addTest(DHCPConfigTest('test_delete_reserved_host')) + suite.addTest(DHCPConfigTest('test_resolve_config_with_hosts')) + suite.addTest(DHCPConfigTest('test_set_subnet_range')) + + runner = unittest.TextTestRunner() + runner.run(suite) diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_lease.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_lease.py new file mode 100644 index 000000000..dd7ba9516 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_lease.py @@ -0,0 +1,75 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary methods to create and monitor DHCP +leases on the server""" +from datetime import datetime +import time + +time_format = '%Y-%m-%d %H:%M:%S' + + +class DHCPLease(object): + """Represents a DHCP Server lease""" + hw_addr = None + ip = None + hostname = None + expires = None + + def __init__(self, lease): + self._make_lease(lease) + + def _make_lease(self, lease): + if lease is not None: + sections_raw = lease.split(' ') + sections = [] + for section in sections_raw: + if section.strip(): + sections.append(section) + self.hw_addr = sections[0] + self.ip = sections[1] + self.hostname = sections[2] + self.expires = sections[3] + ' ' + sections[4] + self.manufacturer = ' '.join(sections[5:]) + + def get_millis(self, timestamp): + dt_obj = datetime.strptime(timestamp, time_format) + millis = dt_obj.timestamp() * 1000 + return millis + + def get_expires_millis(self): + return self.get_millis(self.expires) + + def is_expired(self): + expires_millis = self.get_expires_millis() + cur_time = int(round(time.time()) * 1000) + return cur_time >= expires_millis + + def __str__(self): + lease = {} + if self.hw_addr is not None: + lease['hw_addr'] = self.hw_addr + + if self.ip is not None: + lease['ip'] = self.ip + + if self.hostname is not None: + lease['hostname'] = self.hostname + + if self.expires is not None: + lease['expires'] = self.expires + + if self.manufacturer is not None: + lease['manufacturer'] = self.manufacturer + + return str(lease) diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py new file mode 100644 index 000000000..aa2945759 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py @@ -0,0 +1,107 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Used to resolve the DHCP servers lease information""" +import os +from dhcp_lease import DHCPLease +import logger +from common import util + +LOG_NAME = 'dhcp_lease' +LOGGER = None + +DHCP_LEASE_FILES = [ + '/var/lib/dhcp/dhcpd.leases', '/var/lib/dhcp/dhcpd.leases~', + '/var/lib/dhcp/dhcpd6.leases', '/var/lib/dhcp/dhcpd6.leases~' +] +DHCP_CONFIG_FILE = '/etc/dhcp/dhcpd.conf' + + +class DHCPLeases: + """Leases for the DHCP server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def delete_all_hosts(self): + LOGGER.info('Deleting hosts') + for lease in DHCP_LEASE_FILES: + LOGGER.info('Checking file: ' + lease) + if os.path.exists(lease): + LOGGER.info('File Exists: ' + lease) + try: + # Delete existing lease file + os.remove(lease) + except OSError as e: + LOGGER.info(f'Error occurred while deleting the file: {e}') + # Create an empty lease file + with open(lease, 'w', encoding='UTF-8'): + pass + + def get_lease(self, hw_addr): + for lease in self.get_leases(): + if lease.hw_addr == hw_addr: + return lease + + def get_leases(self): + leases = [] + lease_list_raw = self._get_lease_list() + LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + '\n') + lease_list_start = lease_list_raw.find('=========', 0) + lease_list_start = lease_list_raw.find('\n', lease_list_start) + lease_list = lease_list_raw[lease_list_start + 1:] + lines = lease_list.split('\n') + for line in lines: + try: + lease = DHCPLease(line) + leases.append(lease) + except Exception as e: # pylint: disable=W0718 + # Let non lease lines file without extra checks + LOGGER.error('Making Lease Error: ' + str(e)) + LOGGER.error('Not a valid lease line: ' + line) + return leases + + def delete_lease(self, ip_addr): + LOGGER.info('Deleting lease') + for lease in DHCP_LEASE_FILES: + LOGGER.info('Checking file: ' + lease) + if os.path.exists(lease): + LOGGER.info('File Exists: ' + lease) + try: + # Delete existing lease file + with (open(lease, 'r', encoding='UTF-8')) as f: + contents = f.read() + + while ip_addr in contents: + ix_ip = contents.find(ip_addr) + lease_start = contents.rindex('lease', 0, ix_ip) + lease_end = contents.find('}', lease_start) + LOGGER.info('Lease Location: ' + str(lease_start) + ':' + + str(lease_end)) + contents = contents[0:lease_start] + contents[lease_end + 1:] + + except OSError as e: + LOGGER.info(f'Error occurred while deleting the lease: {e}') + + def _get_lease_list(self): + LOGGER.info('Running lease list command') + try: + result = util.run_command('dhcp-lease-list') + return result[0] + except Exception as e: # pylint: disable=W0718 + LOGGER.error('Error lease list: ' + str(e)) + + def _write_config(self, config): + with open(DHCP_CONFIG_FILE, 'w', encoding='UTF-8') as f: + f.write(config) diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py new file mode 100644 index 000000000..7c2ca3d83 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py @@ -0,0 +1,107 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server""" +import sys +import time +from common import logger +from dhcp_config import DHCPConfig +from radvd_server import RADVDServer +from isc_dhcp_server import ISCDHCPServer + +LOG_NAME = 'dhcp_server' +LOGGER = None + +class DHCPServer: + """Represents the DHCP Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + self.dhcp_config = DHCPConfig() + self.radvd = RADVDServer() + self.isc_dhcp = ISCDHCPServer() + self.dhcp_config.resolve_config() + + def restart(self): + LOGGER.info('Restarting DHCP server') + isc_started = self.isc_dhcp.restart() + radvd_started = self.radvd.restart() + started = isc_started and radvd_started + LOGGER.info('DHCP server restarted: ' + str(started)) + return started + + def start(self): + LOGGER.info('Starting DHCP server') + isc_started = self.isc_dhcp.start() + radvd_started = self.radvd.start() + started = isc_started and radvd_started + LOGGER.info('DHCP server started: ' + str(started)) + return started + + def stop(self): + LOGGER.info('Stopping DHCP server') + isc_stopped = self.isc_dhcp.stop() + radvd_stopped = self.radvd.stop() + stopped = isc_stopped and radvd_stopped + LOGGER.info('DHCP server stopped: ' + str(stopped)) + return stopped + + def is_running(self): + LOGGER.info('Checking DHCP server status') + isc_running = self.isc_dhcp.is_running() + radvd_running = self.radvd.is_running() + running = isc_running and radvd_running + LOGGER.info('DHCP server status: ' + str(running)) + return running + + def boot(self): + LOGGER.info('Booting DHCP server') + booted = False + if self.is_running(): + LOGGER.info('Stopping DHCP server') + stopped = self.stop() + LOGGER.info('DHCP server stopped: ' + str(stopped)) + if self.start(): + # Scan for 5 seconds if not yet ready + for _ in range(5): + time.sleep(1) + booted = self.is_running() + if booted: + break + LOGGER.info('DHCP server booted: ' + str(booted)) + return booted + + +def run(): + dhcp_server = DHCPServer() + booted = dhcp_server.boot() + + if not booted: + LOGGER.error('DHCP server failed to boot. Exiting') + sys.exit(1) + + config = str(dhcp_server.dhcp_config) + while True: + dhcp_server.dhcp_config.resolve_config() + new_config = str(dhcp_server.dhcp_config) + if config != new_config: + LOGGER.info('DHCP server config changed') + config = new_config + dhcp_server.restart() + dhcp_server.radvd.restart() + time.sleep(1) + +if __name__ == '__main__': + run() diff --git a/modules/network/dhcp-1/python/src/grpc_server/isc_dhcp_server.py b/modules/network/dhcp-1/python/src/grpc_server/isc_dhcp_server.py new file mode 100644 index 000000000..1a0e34186 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/isc_dhcp_server.py @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +isc-dhcp server booted from the isc-dhcp service file""" +from common import logger +from common import util + +LOG_NAME = 'isc-dhcp' +LOGGER = None + +class ISCDHCPServer: + """Represents the isc-dhcp server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def restart(self): + LOGGER.info('Restarting isc-dhcp server') + response = util.run_command('isc-dhcp-service restart', False) + LOGGER.info('isc-dhcp server restarted: ' + str(response)) + return response + + def start(self): + LOGGER.info('Starting isc-dhcp server') + response = util.run_command('isc-dhcp-service start', False) + LOGGER.info('isc-dhcp server started: ' + str(response)) + return response + + def stop(self): + LOGGER.info('Stopping isc-dhcp server') + response = util.run_command('isc-dhcp-service stop', False) + LOGGER.info('isc-dhcp server stopped: ' + str(response)) + return response + + def is_running(self): + LOGGER.info('Checking isc-dhcp server') + response = util.run_command('isc-dhcp-service status') + running = response[0] == 'isc-dhcp service is running.' + LOGGER.info('isc-dhcp server status: ' + str(running)) + return running diff --git a/modules/network/dhcp-1/python/src/grpc_server/network_service.py b/modules/network/dhcp-1/python/src/grpc_server/network_service.py new file mode 100644 index 000000000..92726025d --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/network_service.py @@ -0,0 +1,195 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gRPC Network Service for the DHCP Server network module""" +import proto.grpc_pb2_grpc as pb2_grpc +import proto.grpc_pb2 as pb2 + +from dhcp_server import DHCPServer +from dhcp_config import DHCPConfig +from dhcp_leases import DHCPLeases + +import traceback +from common import logger + +LOG_NAME = 'network_service' +LOGGER = None + + +class NetworkService(pb2_grpc.NetworkModule): + """gRPC endpoints for the DHCP Server""" + + def __init__(self): + self._dhcp_server = DHCPServer() + self._dhcp_config = None + self.dhcp_leases = DHCPLeases() + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def _get_dhcp_config(self): + if self._dhcp_config is None: + self._dhcp_config = DHCPConfig() + self._dhcp_config.resolve_config() + return self._dhcp_config + + def RestartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Restarting DHCP server') + try: + started = self._dhcp_server.restart() + LOGGER.info('DHCP server restarted: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to restart DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Starting DHCP server') + try: + started = self._dhcp_server.start() + LOGGER.info('DHCP server started: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to start DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StopDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Stopping DHCP server') + try: + stopped = self._dhcp_server.stop() + LOGGER.info('DHCP server stopped: ' + (str(stopped))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to stop DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def AddReservedLease(self, request, context): # pylint: disable=W0613 + LOGGER.info('Add reserved lease called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.add_reserved_host(request.hostname, request.hw_addr, + request.ip_addr) + dhcp_config.write_config() + LOGGER.info('Reserved lease added') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to add reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DeleteReservedLease(self, request, context): # pylint: disable=W0613 + LOGGER.info('Delete reserved lease called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.delete_reserved_host(request.hw_addr) + dhcp_config.write_config() + LOGGER.info('Reserved lease deleted') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to delete reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DisableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Disable failover called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.disable_failover() + dhcp_config.write_config() + LOGGER.info('Failover disabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to disable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def EnableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Enable failover called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.enable_failover() + dhcp_config.write_config() + LOGGER.info('Failover enabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to enable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetDHCPRange(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP configuration and return + the first range from the first subnet in the file + """ + LOGGER.info('Get DHCP range called') + try: + pool = self._get_dhcp_config()._subnets[0].pools[0] + return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetLease(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP leased address for the + provided MAC address + """ + LOGGER.info('Get lease called') + try: + lease = self.dhcp_leases.get_lease(request.hw_addr) + if lease is not None: + return pb2.Response(code=200, message=str(lease)) + else: + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def SetDHCPRange(self, request, context): # pylint: disable=W0613 + """ + Change DHCP configuration and set the + the first range from the first subnet in the configuration + """ + LOGGER.info('Set DHCP range called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.set_range(request.start, request.end, 0, 0) + dhcp_config.write_config() + LOGGER.info('DHCP range set') + return pb2.Response(code=200, message='DHCP Range Set') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to set DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetStatus(self, request, context): # pylint: disable=W0613 + """ + Return the current status of the network module + """ + dhcp_status = self._dhcp_server.is_running() + message = str({'dhcpStatus': dhcp_status}) + return pb2.Response(code=200, message=message) diff --git a/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto new file mode 100644 index 000000000..e6abda674 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +service NetworkModule { + + rpc RestartDHCPServer(RestartDHCPServerRequest) returns (Response) {}; + + rpc StartDHCPServer(StartDHCPServerRequest) returns (Response) {}; + + rpc StopDHCPServer(StopDHCPServerRequest) returns (Response) {}; + + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; + + rpc DeleteReservedLease(DeleteReservedLeaseRequest) returns (Response) {}; + + rpc DisableFailover(DisableFailoverRequest) returns (Response) {}; + + rpc EnableFailover(EnableFailoverRequest) returns (Response) {}; + + rpc GetDHCPRange(GetDHCPRangeRequest) returns (DHCPRange) {}; + + rpc GetLease(GetLeaseRequest) returns (Response) {}; + + rpc GetStatus(GetStatusRequest) returns (Response) {}; + + rpc SetDHCPRange(SetDHCPRangeRequest) returns (Response) {}; +} + +message AddReservedLeaseRequest { + string hostname = 1; + string hw_addr = 2; + string ip_addr = 3; +} + +message DeleteReservedLeaseRequest { + string hw_addr = 1; +} + +message RestartDHCPServerRequest {} + +message StartDHCPServerRequest {} + +message StopDHCPServerRequest {} + +message DisableFailoverRequest {} + +message EnableFailoverRequest {} + +message GetDHCPRangeRequest {} + +message GetLeaseRequest { + string hw_addr = 1; +} + +message GetStatusRequest {} + +message SetDHCPRangeRequest { + int32 code = 1; + string start = 2; + string end = 3; +} + +message Response { + int32 code = 1; + string message = 2; +} + +message DHCPRange { + int32 code = 1; + string start = 2; + string end = 3; +} \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc_server/radvd_server.py b/modules/network/dhcp-1/python/src/grpc_server/radvd_server.py new file mode 100644 index 000000000..38eec4985 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/radvd_server.py @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +RADVD server booted from the radvd-service file""" +from common import logger +from common import util + +LOG_NAME = 'radvd' +LOGGER = None + +class RADVDServer: + """Represents the RADVD Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def restart(self): + LOGGER.info('Restarting RADVD server') + response = util.run_command('radvd-service restart', False) + LOGGER.info('RADVD restarted: ' + str(response)) + return response + + def start(self): + LOGGER.info('Starting RADVD server') + response = util.run_command('radvd-service start', False) + LOGGER.info('RADVD started: ' + str(response)) + return response + + def stop(self): + LOGGER.info('Stopping RADVD server') + response = util.run_command('radvd-service stop', False) + LOGGER.info('RADVD stopped: ' + str(response)) + return response + + def is_running(self): + LOGGER.info('Checking RADVD status') + response = util.run_command('radvd-service status') + running = response[0] == 'radvd service is running.' + LOGGER.info('RADVD status: ' + str(running)) + return running diff --git a/modules/network/dhcp-2/bin/isc-dhcp-service b/modules/network/dhcp-2/bin/isc-dhcp-service new file mode 100644 index 000000000..ee6df0341 --- /dev/null +++ b/modules/network/dhcp-2/bin/isc-dhcp-service @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CONFIG_FILE=/etc/dhcp/dhcpd.conf +DHCP_PID_FILE=/var/run/dhcpd.pid +DHCP_LOG_FILE=/runtime/network/dhcp2-dhcpd.log + +stop_dhcp(){ + # Directly kill by PID file reference + if [ -f "$DHCP_PID_FILE" ]; then + kill -9 $(cat $DHCP_PID_FILE) || true + rm -f $DHCP_PID_FILE + fi +} + +start_dhcp(){ + /usr/sbin/dhcpd -d &> $DHCP_LOG_FILE & +} + +case "$1" in + start) + start_dhcp + ;; + stop) + stop_dhcp + ;; + restart) + stop_dhcp + sleep 1 + start_dhcp + ;; + status) + if [ -f "$DHCP_PID_FILE" ]; then + echo "isc-dhcp service is running." + else + echo "isc-dhcp service is not running." + fi + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac \ No newline at end of file diff --git a/modules/network/dhcp-2/bin/radvd-service b/modules/network/dhcp-2/bin/radvd-service new file mode 100644 index 000000000..912c64ee3 --- /dev/null +++ b/modules/network/dhcp-2/bin/radvd-service @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RA_PID_FILE=/var/run/radvd/radvd.pid +RA_LOG_FILE=/runtime/network/dhcp2-radvd.log + +stop_radvd(){ + # Directly kill by PID file reference + if [ -f "$RA_PID_FILE" ]; then + kill -9 $(cat $RA_PID_FILE) || true + rm -f $RA_PID_FILE + fi +} + +start_radvd(){ + /usr/sbin/radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE +} + +case "$1" in + start) + start_radvd + ;; + stop) + stop_radvd + ;; + restart) + stop_radvd + sleep 1 + start_radvd + ;; + status) + if [ -f "$RA_PID_FILE" ]; then + echo "radvd service is running." + else + echo "radvd service is not running." + fi + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac \ No newline at end of file diff --git a/modules/network/dhcp-2/bin/start_network_service b/modules/network/dhcp-2/bin/start_network_service index 550854d49..9f702f015 100644 --- a/modules/network/dhcp-2/bin/start_network_service +++ b/modules/network/dhcp-2/bin/start_network_service @@ -20,7 +20,7 @@ DHCP_LOG_FILE=/runtime/network/dhcp2-dhcpd.log RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp2-radvd.log -echo "Starrting Network Service..." +echo "Starting Network Service..." #Enable IPv6 Forwarding sysctl net.ipv6.conf.all.forwarding=1 @@ -29,63 +29,24 @@ sysctl -p # Create leases file if needed touch /var/lib/dhcp/dhcpd.leases -#Create directory for radvd +# Create directory for radvd mkdir /var/run/radvd -#Create and set permissions on the log files +# Create and set permissions on the log files touch $DHCP_LOG_FILE touch $RA_LOG_FILE chown $HOST_USER $DHCP_LOG_FILE chown $HOST_USER $RA_LOG_FILE -#Move the config files to the correct location +# Move the config files to the correct location +cp /testrun/conf/isc-dhcp-server /etc/default/ cp /testrun/conf/dhcpd.conf /etc/dhcp/dhcpd.conf cp /testrun/conf/radvd.conf /etc/radvd.conf -# Restart dhcp server when config changes -while true; do +# Move the service files to the correct location +cp /testrun/bin/isc-dhcp-service /usr/local/bin/ +cp /testrun/bin/radvd-service /usr/local/bin/ - new_checksum=$(md5sum $CONFIG_FILE) - - if [ "$checksum" == "$new_checksum" ]; then - sleep 2 - continue - fi - - echo Config changed. Restarting dhcp server at $(date).. - - if [ -f $DHCP_PID_FILE ]; then - kill -9 $(cat $DHCP_PID_FILE) || true - rm -f $DHCP_PID_FILE - fi - - if [ -f $RA_PID_FILE ]; then - kill -9 $(cat $RA_PID_FILE) || true - rm -f $RA_PID_FILE - fi - - checksum=$new_checksum - - echo Starting isc-dhcp-server at $(date) - - radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE - dhcpd -d &> $DHCP_LOG_FILE & - - while [ ! -f $DHCP_PID_FILE ]; do - echo Waiting for $DHCP_PID_FILE... - sleep 2 - done - - echo $DHCP_PID_FILE now available - - while [ ! -f $RA_PID_FILE ]; do - echo Waiting for $RA_PID_FILE... - sleep 2 - done - - echo $RA_PID_FILE now available - - echo Server now stable - -done \ No newline at end of file +# Start the DHCP Server +python3 -u /testrun/python/src/grpc_server/dhcp_server.py \ No newline at end of file diff --git a/modules/network/dhcp-2/conf/dhcpd.conf b/modules/network/dhcp-2/conf/dhcpd.conf index e73a81441..5a6c82410 100644 --- a/modules/network/dhcp-2/conf/dhcpd.conf +++ b/modules/network/dhcp-2/conf/dhcpd.conf @@ -1,24 +1,26 @@ -default-lease-time 300; - -failover peer "failover-peer" { - secondary; - address 10.10.10.3; - port 647; - peer address 10.10.10.2; - peer port 847; - max-response-delay 60; - max-unacked-updates 10; - load balance max seconds 3; -} - -subnet 10.10.10.0 netmask 255.255.255.0 { - option ntp-servers 10.10.10.5; - option subnet-mask 255.255.255.0; - option broadcast-address 10.10.10.255; - option routers 10.10.10.1; - option domain-name-servers 10.10.10.4; - pool { - failover peer "failover-peer"; - range 10.10.10.10 10.10.10.20; - } -} +default-lease-time 30; +max-lease-time 30; + +failover peer "failover-peer" { + secondary; + address 10.10.10.3; + port 647; + peer address 10.10.10.2; + peer port 847; + max-response-delay 60; + max-unacked-updates 10; + load balance max seconds 3; +} + +subnet 10.10.10.0 netmask 255.255.255.0 { + option ntp-servers 10.10.10.5; + option subnet-mask 255.255.255.0; + option broadcast-address 10.10.10.255; + option routers 10.10.10.1; + option domain-name-servers 10.10.10.4; + interface veth0; + pool { + failover peer "failover-peer"; + range 10.10.10.10 10.10.10.20; + } +} diff --git a/modules/network/dhcp-2/conf/isc-dhcp-server b/modules/network/dhcp-2/conf/isc-dhcp-server new file mode 100644 index 000000000..44db95cd9 --- /dev/null +++ b/modules/network/dhcp-2/conf/isc-dhcp-server @@ -0,0 +1,4 @@ +# On what interfaces should the DHCP server (dhcpd) serve DHCP requests? +# Separate multiple interfaces with spaces, e.g. "eth0 eth1". +INTERFACESv4="veth0" +#INTERFACESv6="veth0" diff --git a/modules/network/dhcp-2/dhcp-2.Dockerfile b/modules/network/dhcp-2/dhcp-2.Dockerfile index df77cb811..153aa50e7 100644 --- a/modules/network/dhcp-2/dhcp-2.Dockerfile +++ b/modules/network/dhcp-2/dhcp-2.Dockerfile @@ -18,8 +18,14 @@ FROM test-run/base:latest ARG MODULE_NAME=dhcp-2 ARG MODULE_DIR=modules/network/$MODULE_NAME +# Install all necessary packages +RUN apt-get install -y wget + +#Update the oui.txt file from ieee +RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ + # Install dhcp server -RUN apt-get install -y isc-dhcp-server radvd +RUN apt-get install -y isc-dhcp-server radvd systemd # Copy over all configuration files COPY $MODULE_DIR/conf /testrun/conf @@ -28,5 +34,4 @@ COPY $MODULE_DIR/conf /testrun/conf COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY $MODULE_DIR/python /testrun/python - +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/network/dhcp-2/python/src/grpc/dhcp_config.py b/modules/network/dhcp-2/python/src/grpc/dhcp_config.py deleted file mode 100644 index f6e79a2ec..000000000 --- a/modules/network/dhcp-2/python/src/grpc/dhcp_config.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains all the necessary classes to maintain the -DHCP server's configuration""" -import re - -CONFIG_FILE = '/etc/dhcp/dhcpd.conf' -CONFIG_FILE_TEST = 'network/modules/dhcp-2/conf/dhcpd.conf' - -DEFAULT_LEASE_TIME_KEY = 'default-lease-time' - - -class DHCPConfig: - """Represents the DHCP Servers configuration and gives access to modify it""" - - def __init__(self): - self._default_lease_time = 300 - self.subnets = [] - self._peer = None - - def write_config(self): - conf = str(self) - print('Writing config: \n' + conf) - with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: - conf_file.write(conf) - - def resolve_config(self): - with open(CONFIG_FILE, 'r', encoding='UTF-8') as f: - conf = f.read() - self.resolve_subnets(conf) - self._peer = DHCPFailoverPeer(conf) - - def resolve_subnets(self, conf): - self.subnets = [] - regex = r'(subnet.*)' - subnets = re.findall(regex, conf, re.MULTILINE | re.DOTALL) - for subnet in subnets: - dhcp_subnet = DHCPSubnet(subnet) - self.subnets.append(dhcp_subnet) - - def set_range(self, start, end, subnet=0, pool=0): - print('Setting Range for pool ') - print(self.subnets[subnet].pools[pool]) - self.subnets[subnet].pools[pool].range_start = start - self.subnets[subnet].pools[pool].range_end = end - - # def resolve_settings(self, conf): - # lines = conf.split('\n') - # for line in lines: - # if DEFAULT_LEASE_TIME_KEY in line: - # self._default_lease_time = line.strip().split( - # DEFAULT_LEASE_TIME_KEY)[1].strip().split(';')[0] - - # self.peer = peer - - def __str__(self): - - config = """\r{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" - - config = config.format(length='multi-line', - DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, - DEFAULT_LEASE_TIME=self._default_lease_time) - - config += '\n\n' + str(self.peer) - for subnet in self._subnets: - config += '\n\n' + str(subnet) - return str(config) - - -FAILOVER_PEER_KEY = 'failover peer' -PRIMARY_KEY = 'primary' -ADDRESS_KEY = 'address' -PORT_KEY = 'port' -PEER_ADDRESS_KEY = 'peer address' -PEER_PORT_KEY = 'peer port' -MAX_RESPONSE_DELAY_KEY = 'max-response-delay' -MAX_UNACKED_UPDATES_KEY = 'max-unacked-updates' -MCLT_KEY = 'mclt' -SPLIT_KEY = 'split' -LOAD_BALANCE_MAX_SECONDS_KEY = 'load balance max seconds' - - -class DHCPFailoverPeer: - """Contains all information to define the DHCP failover peer""" - - def __init__(self, config): - self.name = None - self.primary = False - self.address = None - self.port = None - self.peer_address = None - self.peer_port = None - self.max_response_delay = None - self.max_unacked_updates = None - self.mclt = None - self.split = None - self.load_balance_max_seconds = None - self.peer = None - - self.resolve_peer(config) - - def __str__(self): - config = '{FAILOVER_PEER_KEY} \"{FAILOVER_PEER}\" {{\n' - config += '\tprimary;' if self.primary else 'secondary;' - config += """\n\t{ADDRESS_KEY} {ADDRESS}; - {PORT_KEY} {PORT}; - {PEER_ADDRESS_KEY} {PEER_ADDRESS}; - {PEER_PORT_KEY} {PEER_PORT}; - {MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY}; - {MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES}; - {MCLT_KEY} {MCLT}; - {SPLIT_KEY} {SPLIT}; - {LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS}; - \r}}""" - - return config.format( - length='multi-line', - FAILOVER_PEER_KEY=FAILOVER_PEER_KEY, - FAILOVER_PEER=self.name, - ADDRESS_KEY=ADDRESS_KEY, - ADDRESS=self.address, - PORT_KEY=PORT_KEY, - PORT=self.port, - PEER_ADDRESS_KEY=PEER_ADDRESS_KEY, - PEER_ADDRESS=self.peer_address, - PEER_PORT_KEY=PEER_PORT_KEY, - PEER_PORT=self.peer_port, - MAX_RESPONSE_DELAY_KEY=MAX_RESPONSE_DELAY_KEY, - MAX_RESPONSE_DELAY=self.max_response_delay, - MAX_UNACKED_UPDATES_KEY=MAX_UNACKED_UPDATES_KEY, - MAX_UNACKED_UPDATES=self.max_unacked_updates, - MCLT_KEY=MCLT_KEY, - MCLT=self.mclt, - SPLIT_KEY=SPLIT_KEY, - SPLIT=self.split, - LOAD_BALANCE_MAX_SECONDS_KEY=LOAD_BALANCE_MAX_SECONDS_KEY, - LOAD_BALANCE_MAX_SECONDS=self.load_balance_max_seconds) - - def resolve_peer(self, conf): - peer = '' - lines = conf.split('\n') - for line in lines: - if line.startswith(FAILOVER_PEER_KEY) or len(peer) > 0: - if len(peer) <= 0: - self.name = line.strip().split(FAILOVER_PEER_KEY)[1].strip().split( - '{')[0].split('\"')[1] - peer += line + '\n' - if PRIMARY_KEY in line: - self.primary = True - elif ADDRESS_KEY in line and PEER_ADDRESS_KEY not in line: - self.address = line.strip().split(ADDRESS_KEY)[1].strip().split( - ';')[0] - elif PORT_KEY in line and PEER_PORT_KEY not in line: - self.port = line.strip().split(PORT_KEY)[1].strip().split(';')[0] - elif PEER_ADDRESS_KEY in line: - self.peer_address = line.strip().split( - PEER_ADDRESS_KEY)[1].strip().split(';')[0] - elif PEER_PORT_KEY in line: - self.peer_port = line.strip().split(PEER_PORT_KEY)[1].strip().split( - ';')[0] - elif MAX_RESPONSE_DELAY_KEY in line: - self.max_response_delay = line.strip().split( - MAX_RESPONSE_DELAY_KEY)[1].strip().split(';')[0] - elif MAX_UNACKED_UPDATES_KEY in line: - self.max_unacked_updates = line.strip().split( - MAX_UNACKED_UPDATES_KEY)[1].strip().split(';')[0] - elif MCLT_KEY in line: - self.mclt = line.strip().split(MCLT_KEY)[1].strip().split(';')[0] - elif SPLIT_KEY in line: - self.split = line.strip().split(SPLIT_KEY)[1].strip().split(';')[0] - elif LOAD_BALANCE_MAX_SECONDS_KEY in line: - self.load_balance_max_seconds = line.strip().split( - LOAD_BALANCE_MAX_SECONDS_KEY)[1].strip().split(';')[0] - if line.endswith('}') and len(peer) > 0: - break - self.peer = peer - - -NTP_OPTION_KEY = 'option ntp-servers' -SUBNET_MASK_OPTION_KEY = 'option subnet-mask' -BROADCAST_OPTION_KEY = 'option broadcast-address' -ROUTER_OPTION_KEY = 'option routers' -DNS_OPTION_KEY = 'option domain-name-servers' - - -class DHCPSubnet: - """Represents the DHCP Servers subnet configuration""" - - def __init__(self, subnet): - self._ntp_servers = None - self._subnet_mask = None - self._broadcast = None - self._routers = None - self._dns_servers = None - self.pools = [] - - self.resolve_subnet(subnet) - self.resolve_pools(subnet) - - def __str__(self): - config = """subnet 10.10.10.0 netmask {SUBNET_MASK_OPTION} {{ - \r\t{NTP_OPTION_KEY} {NTP_OPTION}; - \r\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION}; - \r\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION}; - \r\t{ROUTER_OPTION_KEY} {ROUTER_OPTION}; - \r\t{DNS_OPTION_KEY} {DNS_OPTION};""" - - config = config.format(length='multi-line', - NTP_OPTION_KEY=NTP_OPTION_KEY, - NTP_OPTION=self._ntp_servers, - SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, - SUBNET_MASK_OPTION=self._subnet_mask, - BROADCAST_OPTION_KEY=BROADCAST_OPTION_KEY, - BROADCAST_OPTION=self._broadcast, - ROUTER_OPTION_KEY=ROUTER_OPTION_KEY, - ROUTER_OPTION=self._routers, - DNS_OPTION_KEY=DNS_OPTION_KEY, - DNS_OPTION=self._dns_servers) - for pool in self.pools: - config += '\n\t' + str(pool) - - config += '\n\r}' - return config - - def resolve_subnet(self, subnet): - subnet_parts = subnet.split('\n') - for part in subnet_parts: - if NTP_OPTION_KEY in part: - self._ntp_servers = part.strip().split(NTP_OPTION_KEY)[1].strip().split( - ';')[0] - elif SUBNET_MASK_OPTION_KEY in part: - self._subnet_mask = part.strip().split( - SUBNET_MASK_OPTION_KEY)[1].strip().split(';')[0] - elif BROADCAST_OPTION_KEY in part: - self._broadcast = part.strip().split( - BROADCAST_OPTION_KEY)[1].strip().split(';')[0] - elif ROUTER_OPTION_KEY in part: - self._routers = part.strip().split(ROUTER_OPTION_KEY)[1].strip().split( - ';')[0] - elif DNS_OPTION_KEY in part: - self._dns_servers = part.strip().split(DNS_OPTION_KEY)[1].strip().split( - ';')[0] - - def resolve_pools(self, subnet): - regex = r'(pool.*)\}' - pools = re.findall(regex, subnet, re.MULTILINE | re.DOTALL) - for pool in pools: - dhcp_pool = DHCPPool(pool) - self.pools.append(dhcp_pool) - - -FAILOVER_KEY = 'failover peer' -RANGE_KEY = 'range' - - -class DHCPPool: - """Represents a DHCP Servers subnet pool configuration""" - - def __init__(self, pool): - self.failover_peer = None - self.range_start = None - self.range_end = None - self.resolve_pool(pool) - - def __str__(self): - - config = """pool {{ - \r\t\t{FAILOVER_KEY} "{FAILOVER}"; - \r\t\t{RANGE_KEY} {RANGE_START} {RANGE_END}; - \r\t}}""" - - return config.format( - length='multi-line', - FAILOVER_KEY=FAILOVER_KEY, - FAILOVER=self.failover_peer, - RANGE_KEY=RANGE_KEY, - RANGE_START=self.range_start, - RANGE_END=self.range_end, - ) - - def resolve_pool(self, pool): - pool_parts = pool.split('\n') - # pool_parts = pool.split("\n") - for part in pool_parts: - if FAILOVER_KEY in part: - self.failover_peer = part.strip().split(FAILOVER_KEY)[1].strip().split( - ';')[0].replace('\"', '') - if RANGE_KEY in part: - pool_range = part.strip().split(RANGE_KEY)[1].strip().split(';')[0] - self.range_start = pool_range.split(' ')[0].strip() - self.range_end = pool_range.split(' ')[1].strip() diff --git a/modules/network/dhcp-2/python/src/grpc/network_service.py b/modules/network/dhcp-2/python/src/grpc/network_service.py deleted file mode 100644 index 64aab8a07..000000000 --- a/modules/network/dhcp-2/python/src/grpc/network_service.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""gRPC Network Service for the DHCP Server network module""" -import proto.grpc_pb2_grpc as pb2_grpc -import proto.grpc_pb2 as pb2 - -from dhcp_config import DHCPConfig - - -class NetworkService(pb2_grpc.NetworkModule): - """gRPC endpoints for the DHCP Server""" - - def __init__(self): - self._dhcp_config = DHCPConfig() - - def GetDHCPRange(self, request, context): # pylint: disable=W0613 - """ - Resolve the current DHCP configuration and return - the first range from the first subnet in the file - """ - self._dhcp_config.resolve_config() - pool = self._dhcp_config.subnets[0].pools[0] - return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) - - def SetDHCPRange(self, request, context): # pylint: disable=W0613 - """ - Change DHCP configuration and set the - the first range from the first subnet in the configuration - """ - - print('Setting DHCPRange') - print('Start: ' + request.start) - print('End: ' + request.end) - self._dhcp_config.resolve_config() - self._dhcp_config.set_range(request.start, request.end, 0, 0) - self._dhcp_config.write_config() - return pb2.Response(code=200, message='DHCP Range Set') - - def GetStatus(self, request, context): # pylint: disable=W0613 - """ - Return the current status of the network module - """ - # ToDo: Figure out how to resolve the current DHCP status - dhcp_status = True - message = str({'dhcpStatus': dhcp_status}) - return pb2.Response(code=200, message=message) diff --git a/modules/network/dhcp-2/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-2/python/src/grpc/proto/grpc.proto deleted file mode 100644 index 8e2732620..000000000 --- a/modules/network/dhcp-2/python/src/grpc/proto/grpc.proto +++ /dev/null @@ -1,36 +0,0 @@ -syntax = "proto3"; - -service NetworkModule { - - rpc GetDHCPRange(GetDHCPRangeRequest) returns (DHCPRange) {}; - - rpc SetDHCPRange(DHCPRange) returns (Response) {}; - - rpc GetStatus(GetStatusRequest) returns (Response) {}; - - rpc GetIPAddress(GetIPAddressRequest) returns (Response) {}; - - rpc SetLeaseAddress(SetLeaseAddressRequest) returns (Response) {}; - -} - -message Response { - int32 code = 1; - string message = 2; -} - -message DHCPRange { - int32 code = 1; - string start = 2; - string end = 3; -} - -message GetDHCPRangeRequest {} - -message GetIPAddressRequest {} - -message GetStatusRequest {} - -message SetLeaseAddressRequest { - string ipAddress = 1; -} \ No newline at end of file diff --git a/modules/network/dhcp-2/python/src/grpc/__init__.py b/modules/network/dhcp-2/python/src/grpc_server/__init__.py similarity index 100% rename from modules/network/dhcp-2/python/src/grpc/__init__.py rename to modules/network/dhcp-2/python/src/grpc_server/__init__.py diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py new file mode 100644 index 000000000..5357ba7ed --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py @@ -0,0 +1,553 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server's configuration""" +import re +from common import logger + +LOG_NAME = 'dhcp_config' +LOGGER = None + +CONFIG_FILE = '/etc/dhcp/dhcpd.conf' + +DEFAULT_LEASE_TIME_KEY = 'default-lease-time' +MAX_LEASE_TIME_KEY = 'max-lease-time' + + +class DHCPConfig: + """Represents the DHCP Servers configuration and gives access to modify it""" + + def __init__(self): + self._default_lease_time = 30 + self._max_lease_time = 30 + self._subnets = [] + self._peer = None + self._reserved_hosts = [] + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def add_reserved_host(self, hostname, hw_addr, ip_addr): + host = DHCPReservedHost(hostname=hostname, + hw_addr=hw_addr, + fixed_addr=ip_addr) + self._reserved_hosts.append(host) + + def delete_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host.hw_addr: + self._reserved_hosts.remove(host) + + def disable_failover(self): + self._peer.disable() + for subnet in self._subnets: + subnet.disable_peer() + + def enable_failover(self): + self._peer.enable() + for subnet in self._subnets: + subnet.enable_peer() + + def get_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host.hw_addr: + return host + + def write_config(self, config=None): + if config is None: + conf = str(self) + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(conf) + else: + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(config) + + def _get_config(self, config_file=CONFIG_FILE): + content = None + with open(config_file, 'r', encoding='UTF-8') as f: + content = f.read() + return content + + def make(self, conf): + try: + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: # pylint: disable=W0718 + print('Failed to make DHCPConfig: ' + str(e)) + + def resolve_config(self, config_file=CONFIG_FILE): + try: + conf = self._get_config(config_file) + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: # pylint: disable=W0718 + print('Failed to resolve config: ' + str(e)) + + def resolve_subnets(self, conf): + subnets = [] + regex = r'(subnet.*)' + subnets_conf = re.findall(regex, conf, re.MULTILINE | re.DOTALL) + for subnet in subnets_conf: + dhcp_subnet = DHCPSubnet(subnet) + subnets.append(dhcp_subnet) + return subnets + + def resolve_reserved_hosts(self, conf): + hosts = [] + host_start = 0 + while True: + host_start = conf.find('host', host_start) + if host_start < 0: + break + else: + host_end = conf.find('}', host_start) + host = DHCPReservedHost(config=conf[host_start:host_end + 1]) + hosts.append(host) + host_start = host_end + 1 + return hosts + + def set_range(self, start, end, subnet=0, pool=0): + # Calculate the subnet from the range + octets = start.split('.') + octets[-1] = '0' + dhcp_subnet = '.'.join(octets) + + # Calcualte the netmask from the range + prefix = self.calculate_prefix_length(start, end) + netmask = self.calculate_netmask(prefix) + + #Update the subnet, range and netmask + self._subnets[subnet].set_subnet(dhcp_subnet, netmask) + self._subnets[subnet].pools[pool].set_range(start, end) + + def calculate_prefix_length(self, start_ip, end_ip): + start_octets = start_ip.split('.') + end_octets = end_ip.split('.') + + start_int = int( + ''.join(format(int(octet), '08b') for octet in start_octets), 2) + end_int = int(''.join(format(int(octet), '08b') for octet in end_octets), 2) + + xor_result = start_int ^ end_int + prefix_length = 32 - xor_result.bit_length() + + return prefix_length + + def calculate_netmask(self, prefix_length): + num_network_bits = prefix_length + num_host_bits = 32 - num_network_bits + + netmask_int = (2**num_network_bits - 1) << num_host_bits + netmask_octets = [(netmask_int >> (i * 8)) & 0xff for i in range(3, -1, -1)] + + return '.'.join(str(octet) for octet in netmask_octets) + + def __str__(self): + + config = ('{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};' + if self._default_lease_time is not None else '') + config += ('\n\r{MAX_LEASE_TIME_KEY} {MAX_LEASE_TIME};' + if self._max_lease_time is not None else '') + + # Encode the top level config options + #config = """{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" + config = config.format(length='multi-line', + DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, + DEFAULT_LEASE_TIME=self._default_lease_time, + MAX_LEASE_TIME_KEY=MAX_LEASE_TIME_KEY, + MAX_LEASE_TIME=self._max_lease_time) + + # Encode the failover peer + config += '\n\n' + str(self._peer) + + # Encode the subnets + for subnet in self._subnets: + config += '\n\n' + str(subnet) + + # Encode the reserved hosts + for host in self._reserved_hosts: + config += '\n' + str(host) + + return str(config) + + +FAILOVER_PEER_KEY = 'failover peer' +PRIMARY_KEY = 'primary' +ADDRESS_KEY = 'address' +PORT_KEY = 'port' +PEER_ADDRESS_KEY = 'peer address' +PEER_PORT_KEY = 'peer port' +MAX_RESPONSE_DELAY_KEY = 'max-response-delay' +MAX_UNACKED_UPDATES_KEY = 'max-unacked-updates' +MCLT_KEY = 'mclt' +SPLIT_KEY = 'split' +LOAD_BALANCE_MAX_SECONDS_KEY = 'load balance max seconds' + + +class DHCPFailoverPeer: + """Contains all information to define the DHCP failover peer""" + + def __init__(self, config): + self.name = None + self.primary = False + self.address = None + self.port = None + self.peer_address = None + self.peer_port = None + self.max_response_delay = None + self.max_unacked_updates = None + self.mclt = None + self.split = None + self.load_balance_max_seconds = None + self.peer = None + self.enabled = True + + self.resolve_peer(config) + + def __str__(self): + config = '{FAILOVER_PEER_KEY} \"{FAILOVER_PEER}\" {{\n' + config += '\tprimary;' if self.primary else 'secondary;' + config += '\n\t{ADDRESS_KEY} {ADDRESS};' if self.address is not None else '' + config += '\n\t{PORT_KEY} {PORT};' if self.port is not None else '' + config += ('\n\t{PEER_ADDRESS_KEY} {PEER_ADDRESS};' + if self.peer_address is not None else '') + config += ('\n\t{PEER_PORT_KEY} {PEER_PORT};' + if self.peer_port is not None else '') + config += ('\n\t{MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY};' + if self.max_response_delay is not None else '') + config += ('\n\t{MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES};' + if self.max_unacked_updates is not None else '') + config += '\n\t{MCLT_KEY} {MCLT};' if self.mclt is not None else '' + config += '\n\t{SPLIT_KEY} {SPLIT};' if self.split is not None else '' + config += ('\n\t{LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS};' + if self.load_balance_max_seconds is not None else '') + config += '\n\r}}' + + config = config.format( + length='multi-line', + FAILOVER_PEER_KEY=FAILOVER_PEER_KEY, + FAILOVER_PEER=self.name, + ADDRESS_KEY=ADDRESS_KEY, + ADDRESS=self.address, + PORT_KEY=PORT_KEY, + PORT=self.port, + PEER_ADDRESS_KEY=PEER_ADDRESS_KEY, + PEER_ADDRESS=self.peer_address, + PEER_PORT_KEY=PEER_PORT_KEY, + PEER_PORT=self.peer_port, + MAX_RESPONSE_DELAY_KEY=MAX_RESPONSE_DELAY_KEY, + MAX_RESPONSE_DELAY=self.max_response_delay, + MAX_UNACKED_UPDATES_KEY=MAX_UNACKED_UPDATES_KEY, + MAX_UNACKED_UPDATES=self.max_unacked_updates, + MCLT_KEY=MCLT_KEY, + MCLT=self.mclt, + SPLIT_KEY=SPLIT_KEY, + SPLIT=self.split, + LOAD_BALANCE_MAX_SECONDS_KEY=LOAD_BALANCE_MAX_SECONDS_KEY, + LOAD_BALANCE_MAX_SECONDS=self.load_balance_max_seconds) + + if not self.enabled: + lines = config.strip().split('\n') + for i in range(len(lines) - 1): + lines[i] = '#' + lines[i] + lines[-1] = '#' + lines[-1].strip() # Handle the last line separately + config = '\n'.join(lines) + + return config + + def disable(self): + self.enabled = False + + def enable(self): + self.enabled = True + + def resolve_peer(self, conf): + peer = '' + lines = conf.split('\n') + for line in lines: + if line.startswith(FAILOVER_PEER_KEY) or len(peer) > 0: + if len(peer) <= 0: + self.name = line.strip().split(FAILOVER_PEER_KEY)[1].strip().split( + '{')[0].split('\"')[1] + peer += line + '\n' + if PRIMARY_KEY in line: + self.primary = True + elif ADDRESS_KEY in line and PEER_ADDRESS_KEY not in line: + self.address = line.strip().split(ADDRESS_KEY)[1].strip().split( + ';')[0] + elif PORT_KEY in line and PEER_PORT_KEY not in line: + self.port = line.strip().split(PORT_KEY)[1].strip().split(';')[0] + elif PEER_ADDRESS_KEY in line: + self.peer_address = line.strip().split( + PEER_ADDRESS_KEY)[1].strip().split(';')[0] + elif PEER_PORT_KEY in line: + self.peer_port = line.strip().split(PEER_PORT_KEY)[1].strip().split( + ';')[0] + elif MAX_RESPONSE_DELAY_KEY in line: + self.max_response_delay = line.strip().split( + MAX_RESPONSE_DELAY_KEY)[1].strip().split(';')[0] + elif MAX_UNACKED_UPDATES_KEY in line: + self.max_unacked_updates = line.strip().split( + MAX_UNACKED_UPDATES_KEY)[1].strip().split(';')[0] + elif MCLT_KEY in line: + self.mclt = line.strip().split(MCLT_KEY)[1].strip().split(';')[0] + elif SPLIT_KEY in line: + self.split = line.strip().split(SPLIT_KEY)[1].strip().split(';')[0] + elif LOAD_BALANCE_MAX_SECONDS_KEY in line: + self.load_balance_max_seconds = line.strip().split( + LOAD_BALANCE_MAX_SECONDS_KEY)[1].strip().split(';')[0] + if line.endswith('}') and len(peer) > 0: + break + self.peer = peer + + +SUBNET_KEY = 'subnet' +NTP_OPTION_KEY = 'option ntp-servers' +SUBNET_MASK_OPTION_KEY = 'option subnet-mask' +BROADCAST_OPTION_KEY = 'option broadcast-address' +ROUTER_OPTION_KEY = 'option routers' +DNS_OPTION_KEY = 'option domain-name-servers' +INTERFACE_KEY = 'interface' +AUTHORITATIVE_KEY = 'authoritative' + + +class DHCPSubnet: + """Represents the DHCP Servers subnet configuration""" + + def __init__(self, subnet): + self._authoritative = False + self._subnet = None + self._ntp_servers = None + self._subnet_mask = None + self._broadcast = None + self._routers = None + self._dns_servers = None + self._interface = None + self.pools = [] + + self.resolve_subnet(subnet) + self.resolve_pools(subnet) + + def __str__(self): + config = 'subnet {SUBNET_OPTION} netmask {SUBNET_MASK_OPTION} {{' + config += ('\n\t{NTP_OPTION_KEY} {NTP_OPTION};' + if self._ntp_servers is not None else '') + config += ('\n\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION};' + if self._subnet_mask is not None else '') + config += ('\n\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION};' + if self._broadcast is not None else '') + config += ('\n\t{ROUTER_OPTION_KEY} {ROUTER_OPTION};' + if self._routers is not None else '') + config += ('\n\t{DNS_OPTION_KEY} {DNS_OPTION};' + if self._dns_servers is not None else '') + config += ('\n\t{INTERFACE_KEY} {INTERFACE_OPTION};' + if self._interface is not None else '') + config += '\n\t{AUTHORITATIVE_KEY};' if self._authoritative else '' + + config = config.format(length='multi-line', + SUBNET_OPTION=self._subnet, + NTP_OPTION_KEY=NTP_OPTION_KEY, + NTP_OPTION=self._ntp_servers, + SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, + SUBNET_MASK_OPTION=self._subnet_mask, + BROADCAST_OPTION_KEY=BROADCAST_OPTION_KEY, + BROADCAST_OPTION=self._broadcast, + ROUTER_OPTION_KEY=ROUTER_OPTION_KEY, + ROUTER_OPTION=self._routers, + DNS_OPTION_KEY=DNS_OPTION_KEY, + DNS_OPTION=self._dns_servers, + INTERFACE_KEY=INTERFACE_KEY, + INTERFACE_OPTION=self._interface, + AUTHORITATIVE_KEY=AUTHORITATIVE_KEY) + + # if not self._authoritative: + # config = config.replace(AUTHORITATIVE_KEY, '#' + AUTHORITATIVE_KEY) + + for pool in self.pools: + config += '\n\t' + str(pool) + + config += '\n}' + return config + + def disable_peer(self): + for pool in self.pools: + pool.disable_peer() + + def enable_peer(self): + for pool in self.pools: + pool.enable_peer() + + def set_subnet(self, subnet, netmask=None): + if netmask is None: + netmask = '255.255.255.0' + self._subnet = subnet + self._subnet_mask = netmask + + # Calculate the broadcast from the subnet and netmask + broadcast = self.calculate_broadcast_address(subnet, netmask) + self._broadcast = broadcast + + def calculate_broadcast_address(self, subnet_address, netmask): + subnet_octets = subnet_address.split('.') + netmask_octets = netmask.split('.') + + subnet_int = int( + ''.join(format(int(octet), '08b') for octet in subnet_octets), 2) + netmask_int = int( + ''.join(format(int(octet), '08b') for octet in netmask_octets), 2) + + broadcast_int = subnet_int | (~netmask_int & 0xffffffff) + broadcast_octets = [(broadcast_int >> (i * 8)) & 0xff + for i in range(3, -1, -1)] + + return '.'.join(str(octet) for octet in broadcast_octets) + + def resolve_subnet(self, subnet): + subnet_parts = subnet.split('\n') + for part in subnet_parts: + if part.strip().startswith(SUBNET_KEY): + self._subnet = part.strip().split()[1] + elif NTP_OPTION_KEY in part: + self._ntp_servers = part.strip().split(NTP_OPTION_KEY)[1].strip().split( + ';')[0] + elif SUBNET_MASK_OPTION_KEY in part: + self._subnet_mask = part.strip().split( + SUBNET_MASK_OPTION_KEY)[1].strip().split(';')[0] + elif BROADCAST_OPTION_KEY in part: + self._broadcast = part.strip().split( + BROADCAST_OPTION_KEY)[1].strip().split(';')[0] + elif ROUTER_OPTION_KEY in part: + self._routers = part.strip().split(ROUTER_OPTION_KEY)[1].strip().split( + ';')[0] + elif DNS_OPTION_KEY in part: + self._dns_servers = part.strip().split(DNS_OPTION_KEY)[1].strip().split( + ';')[0] + elif INTERFACE_KEY in part: + self._interface = part.strip().split(INTERFACE_KEY)[1].strip().split( + ';')[0] + elif AUTHORITATIVE_KEY in part: + self._authoritative = True + + def resolve_pools(self, subnet): + regex = r'(pool.*)\}' + pools = re.findall(regex, subnet, re.MULTILINE | re.DOTALL) + for pool in pools: + dhcp_pool = DHCPPool(pool) + self.pools.append(dhcp_pool) + + +FAILOVER_KEY = 'failover peer' +RANGE_KEY = 'range' + + +class DHCPPool: + """Represents a DHCP Servers subnet pool configuration""" + + def __init__(self, pool): + self.failover_peer = None + self.range_start = None + self.range_end = None + self.resolve_pool(pool) + self._peer_enabled = True + + def __str__(self): + config = 'pool {{' + config += ('\n\t\t{FAILOVER_KEY} "{FAILOVER}";' + if self.failover_peer is not None else '') + config += ('\n\t\t{RANGE_KEY} {RANGE_START} {RANGE_END};' + if self.range_start is not None and self.range_end is not None + else '') + config += '\n\t}}' + + config = config.format( + length='multi-line', + FAILOVER_KEY=FAILOVER_KEY, + FAILOVER=self.failover_peer, + RANGE_KEY=RANGE_KEY, + RANGE_START=self.range_start, + RANGE_END=self.range_end, + ) + + if not self._peer_enabled: + config = config.replace(FAILOVER_KEY, '#' + FAILOVER_KEY) + + return config + + def disable_peer(self): + self._peer_enabled = False + + def enable_peer(self): + self._peer_enabled = True + + def set_range(self, start, end): + self.range_start = start + self.range_end = end + + def resolve_pool(self, pool): + pool_parts = pool.split('\n') + for part in pool_parts: + if FAILOVER_KEY in part: + self.failover_peer = part.strip().split(FAILOVER_KEY)[1].strip().split( + ';')[0].replace('\"', '') + if RANGE_KEY in part: + pool_range = part.strip().split(RANGE_KEY)[1].strip().split(';')[0] + self.range_start = pool_range.split(' ')[0].strip() + self.range_end = pool_range.split(' ')[1].strip() + + +HOST_KEY = 'host' +HARDWARE_KEY = 'hardware ethernet' +FIXED_ADDRESS_KEY = 'fixed-address' + + +class DHCPReservedHost: + """Represents a DHCP Servers subnet pool configuration""" + + def __init__(self, hostname=None, hw_addr=None, fixed_addr=None, config=None): + if config is None: + self.host = hostname + self.hw_addr = hw_addr + self.fixed_addr = fixed_addr + else: + self.resolve_host(config) + + def __str__(self): + + config = """{HOST_KEY} {HOSTNAME} {{ + \r\t{HARDWARE_KEY} {HW_ADDR}; + \r\t{FIXED_ADDRESS_KEY} {RESERVED_IP}; + \r}}""" + + config = config.format( + length='multi-line', + HOST_KEY=HOST_KEY, + HOSTNAME=self.host, + HARDWARE_KEY=HARDWARE_KEY, + HW_ADDR=self.hw_addr, + FIXED_ADDRESS_KEY=FIXED_ADDRESS_KEY, + RESERVED_IP=self.fixed_addr, + ) + return config + + def resolve_host(self, reserved_host): + host_parts = reserved_host.split('\n') + for part in host_parts: + if HOST_KEY in part: + self.host = part.strip().split(HOST_KEY)[1].strip().split('{')[0] + elif HARDWARE_KEY in part: + self.hw_addr = part.strip().split(HARDWARE_KEY)[1].strip().split(';')[0] + elif FIXED_ADDRESS_KEY in part: + self.fixed_addr = part.strip().split( + FIXED_ADDRESS_KEY)[1].strip().split(';')[0] diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py new file mode 100644 index 000000000..0a156db68 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py @@ -0,0 +1,112 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit Testing for the DHCP Server config""" +import unittest +from dhcp_config import DHCPConfig +import os + +CONFIG_FILE = 'conf/dhcpd.conf' +DHCP_CONFIG = None + +def get_config_file_path(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + module_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(current_dir)))) + conf_file = os.path.join(module_dir, CONFIG_FILE) + return conf_file + +def get_config(): + dhcp_config = DHCPConfig() + dhcp_config.resolve_config(get_config_file_path()) + return dhcp_config + +class DHCPConfigTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + # Resolve the config + global DHCP_CONFIG + DHCP_CONFIG = get_config() + + def test_resolve_config(self): + print('Test Resolve Config:\n' + str(DHCP_CONFIG)) + + # Resolve the raw config file + with open(get_config_file_path(), 'r', encoding='UTF-8') as f: + lines = f.readlines() + + # Get the resolved config as a + conf_parts = str(DHCP_CONFIG).split('\n') + + # dhcpd conf is not picky about spacing so we just + # need to check contents of each line for matching + # to make sure evertying matches + for i in range(len(lines)): + self.assertEqual(lines[i].strip(), conf_parts[i].strip()) + + def test_disable_failover(self): + DHCP_CONFIG.disable_failover() + print('Test Disable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertTrue(line.startswith('#')) + + def test_enable_failover(self): + DHCP_CONFIG.enable_failover() + print('Test Enable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertFalse(line.startswith('#')) + + def test_add_reserved_host(self): + DHCP_CONFIG.add_reserved_host('test', '00:11:22:33:44:55', '192.168.10.5') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print('AddHostConfig:\n' + str(DHCP_CONFIG)) + + def test_delete_reserved_host(self): + DHCP_CONFIG.delete_reserved_host('00:11:22:33:44:55') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNone(host) + print('DeleteHostConfig:\n' + str(DHCP_CONFIG)) + + def test_resolve_config_with_hosts(self): + DHCP_CONFIG.add_reserved_host('test', '00:11:22:33:44:55', '192.168.10.5') + config_with_hosts = DHCPConfig() + config_with_hosts.make(str(DHCP_CONFIG)) + host = config_with_hosts.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print('ResolveConfigWithHosts:\n' + str(config_with_hosts)) + + def test_set_subnet_range(self): + range_start = '10.0.0.100' + range_end = '10.0.0.200' + DHCP_CONFIG.set_range(range_start, range_end) + subnets = DHCP_CONFIG.resolve_subnets(str(DHCP_CONFIG)) + pool = subnets[0].pools[0] + self.assertTrue(pool.range_start == range_start + and pool.range_end == range_end) + print('SetSubnetRange:\n' + str(DHCP_CONFIG)) + +if __name__ == '__main__': + suite = unittest.TestSuite() + suite.addTest(DHCPConfigTest('test_resolve_config')) + suite.addTest(DHCPConfigTest('test_disable_failover')) + suite.addTest(DHCPConfigTest('test_enable_failover')) + suite.addTest(DHCPConfigTest('test_add_reserved_host')) + suite.addTest(DHCPConfigTest('test_delete_reserved_host')) + suite.addTest(DHCPConfigTest('test_resolve_config_with_hosts')) + suite.addTest(DHCPConfigTest('test_set_subnet_range')) + runner = unittest.TextTestRunner() + runner.run(suite) diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py new file mode 100644 index 000000000..dd7ba9516 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py @@ -0,0 +1,75 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary methods to create and monitor DHCP +leases on the server""" +from datetime import datetime +import time + +time_format = '%Y-%m-%d %H:%M:%S' + + +class DHCPLease(object): + """Represents a DHCP Server lease""" + hw_addr = None + ip = None + hostname = None + expires = None + + def __init__(self, lease): + self._make_lease(lease) + + def _make_lease(self, lease): + if lease is not None: + sections_raw = lease.split(' ') + sections = [] + for section in sections_raw: + if section.strip(): + sections.append(section) + self.hw_addr = sections[0] + self.ip = sections[1] + self.hostname = sections[2] + self.expires = sections[3] + ' ' + sections[4] + self.manufacturer = ' '.join(sections[5:]) + + def get_millis(self, timestamp): + dt_obj = datetime.strptime(timestamp, time_format) + millis = dt_obj.timestamp() * 1000 + return millis + + def get_expires_millis(self): + return self.get_millis(self.expires) + + def is_expired(self): + expires_millis = self.get_expires_millis() + cur_time = int(round(time.time()) * 1000) + return cur_time >= expires_millis + + def __str__(self): + lease = {} + if self.hw_addr is not None: + lease['hw_addr'] = self.hw_addr + + if self.ip is not None: + lease['ip'] = self.ip + + if self.hostname is not None: + lease['hostname'] = self.hostname + + if self.expires is not None: + lease['expires'] = self.expires + + if self.manufacturer is not None: + lease['manufacturer'] = self.manufacturer + + return str(lease) diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py new file mode 100644 index 000000000..08e6feabe --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py @@ -0,0 +1,107 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Used to resolve the DHCP servers lease information""" +import os +from dhcp_lease import DHCPLease +import logger +from common import util + +LOG_NAME = 'dhcp_lease' +LOGGER = None + +DHCP_LEASE_FILES = [ + '/var/lib/dhcp/dhcpd.leases', '/var/lib/dhcp/dhcpd.leases~', + '/var/lib/dhcp/dhcpd6.leases', '/var/lib/dhcp/dhcpd6.leases~' +] +DHCP_CONFIG_FILE = '/etc/dhcp/dhcpd.conf' + + +class DHCPLeases: + """Leases for the DHCP server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def delete_all_hosts(self): + LOGGER.info('Deleting hosts') + for lease in DHCP_LEASE_FILES: + LOGGER.info('Checking file: ' + lease) + if os.path.exists(lease): + LOGGER.info('File Exists: ' + lease) + try: + # Delete existing lease file + os.remove(lease) + except OSError as e: + LOGGER.info(f'Error occurred while deleting the file: {e}') + # Create an empty lease file + with open(lease, 'w', encoding='UTF-8'): + pass + + def get_lease(self, hw_addr): + for lease in self.get_leases(): + if lease.hw_addr == hw_addr: + return lease + + def get_leases(self): + leases = [] + lease_list_raw = self._get_lease_list() + LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + '\n') + lease_list_start = lease_list_raw.find('=========',0) + lease_list_start = lease_list_raw.find('\n',lease_list_start) + lease_list = lease_list_raw[lease_list_start+1:] + lines = lease_list.split('\n') + for line in lines: + try: + lease = DHCPLease(line) + leases.append(lease) + except Exception as e: # pylint: disable=W0718 + # Let non lease lines file without extra checks + LOGGER.error('Making Lease Error: ' + str(e)) + LOGGER.error('Not a valid lease line: ' + line) + return leases + + def delete_lease(self, ip_addr): + LOGGER.info('Deleting lease') + for lease in DHCP_LEASE_FILES: + LOGGER.info('Checking file: ' + lease) + if os.path.exists(lease): + LOGGER.info('File Exists: ' + lease) + try: + # Delete existing lease file + with (open(lease, 'r', encoding='UTF-8')) as f: + contents = f.read() + + while ip_addr in contents: + ix_ip = contents.find(ip_addr) + lease_start = contents.rindex('lease', 0, ix_ip) + lease_end = contents.find('}', lease_start) + LOGGER.info('Lease Location: ' + str(lease_start) + ':' + + str(lease_end)) + contents = contents[0:lease_start] + contents[lease_end + 1:] + + except OSError as e: + LOGGER.info(f'Error occurred while deleting the lease: {e}') + + def _get_lease_list(self): + LOGGER.info('Running lease list command') + try: + result = util.run_command('dhcp-lease-list') + return result[0] + except Exception as e: # pylint: disable=W0718 + LOGGER.error('Error lease list: ' + str(e)) + + def _write_config(self, config): + with open(DHCP_CONFIG_FILE, 'w', encoding='UTF-8') as f: + f.write(config) diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py new file mode 100644 index 000000000..7aec674f2 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py @@ -0,0 +1,106 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server""" +import sys +import time +from common import logger +from dhcp_config import DHCPConfig +from radvd_server import RADVDServer +from isc_dhcp_server import ISCDHCPServer + +LOG_NAME = 'dhcp_server' +LOGGER = None + +class DHCPServer: + """Represents the DHCP Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + self.dhcp_config = DHCPConfig() + self.radvd = RADVDServer() + self.isc_dhcp = ISCDHCPServer() + self.dhcp_config.resolve_config() + + def restart(self): + LOGGER.info('Restarting DHCP server') + isc_started = self.isc_dhcp.restart() + radvd_started = self.radvd.restart() + started = isc_started and radvd_started + LOGGER.info('DHCP server restarted: ' + str(started)) + return started + + def start(self): + LOGGER.info('Starting DHCP server') + isc_started = self.isc_dhcp.start() + radvd_started = self.radvd.start() + started = isc_started and radvd_started + LOGGER.info('DHCP server started: ' + str(started)) + return started + + def stop(self): + LOGGER.info('Stopping DHCP server') + isc_stopped = self.isc_dhcp.stop() + radvd_stopped = self.radvd.stop() + stopped = isc_stopped and radvd_stopped + LOGGER.info('DHCP server stopped: ' + str(stopped)) + return stopped + + def is_running(self): + LOGGER.info('Checking DHCP server status') + isc_running = self.isc_dhcp.is_running() + radvd_running = self.radvd.is_running() + running = isc_running and radvd_running + LOGGER.info('DHCP server status: ' + str(running)) + return running + + def boot(self): + LOGGER.info('Booting DHCP server') + booted = False + if self.is_running(): + LOGGER.info('Stopping DHCP server') + stopped = self.stop() + LOGGER.info('DHCP server stopped: ' + str(stopped)) + if self.start(): + # Scan for 5 seconds if not yet ready + for _ in range(5): + time.sleep(1) + booted = self.is_running() + if booted: + break + LOGGER.info('DHCP server booted: ' + str(booted)) + return booted + +def run(): + dhcp_server = DHCPServer() + booted = dhcp_server.boot() + + if not booted: + LOGGER.error('DHCP server failed to boot. Exiting') + sys.exit(1) + + config = str(dhcp_server.dhcp_config) + while True: + dhcp_server.dhcp_config.resolve_config() + new_config = str(dhcp_server.dhcp_config) + if config != new_config: + LOGGER.info('DHCP server config changed') + config = new_config + dhcp_server.restart() + dhcp_server.radvd.restart() + time.sleep(1) + +if __name__ == '__main__': + run() diff --git a/modules/network/dhcp-2/python/src/grpc_server/isc_dhcp_server.py b/modules/network/dhcp-2/python/src/grpc_server/isc_dhcp_server.py new file mode 100644 index 000000000..429c06da0 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/isc_dhcp_server.py @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +isc-dhcp server booted from the isc-dhcp service file""" +from common import logger +from common import util + +LOG_NAME = 'isc-dhcp' +LOGGER = None + +class ISCDHCPServer: + """Represents the isc-dhcp server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def restart(self): + LOGGER.info('Restarting isc-dhcp server') + response = util.run_command('isc-dhcp-service restart', False) + LOGGER.info('isc-dhcp server restarted: ' + str(response)) + return response + + def start(self): + LOGGER.info('Starting isc-dhcp server') + response = util.run_command('isc-dhcp-service start', False) + LOGGER.info('isc-dhcp server started: ' + str(response)) + return response + + def stop(self): + LOGGER.info('Stopping isc-dhcp server') + response = util.run_command('isc-dhcp-service stop', False) + LOGGER.info('isc-dhcp server stopped: ' + str(response)) + return response + + def is_running(self): + LOGGER.info('Checking isc-dhcp server') + response = util.run_command('isc-dhcp-service status') + running = response[0] == 'isc-dhcp service is running.' + LOGGER.info('isc-dhcp server status: ' + str(running)) + return running diff --git a/modules/network/dhcp-2/python/src/grpc_server/network_service.py b/modules/network/dhcp-2/python/src/grpc_server/network_service.py new file mode 100644 index 000000000..f9deba965 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/network_service.py @@ -0,0 +1,195 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gRPC Network Service for the DHCP Server network module""" +import proto.grpc_pb2_grpc as pb2_grpc +import proto.grpc_pb2 as pb2 + +from dhcp_server import DHCPServer +from dhcp_config import DHCPConfig +from dhcp_leases import DHCPLeases + +import traceback +from common import logger + +LOG_NAME = 'network_service' +LOGGER = None + + +class NetworkService(pb2_grpc.NetworkModule): + """gRPC endpoints for the DHCP Server""" + + def __init__(self): + self._dhcp_server = DHCPServer() + self._dhcp_config = None + self.dhcp_leases = DHCPLeases() + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def _get_dhcp_config(self): + if self._dhcp_config is None: + self._dhcp_config = DHCPConfig() + self._dhcp_config.resolve_config() + return self._dhcp_config + + def RestartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Restarting DHCP server') + try: + started = self._dhcp_server.restart() + LOGGER.info('DHCP server restarted: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to restart DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Starting DHCP server') + try: + started = self._dhcp_server.start() + LOGGER.info('DHCP server started: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to start DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StopDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Stopping DHCP server') + try: + stopped = self._dhcp_server.stop() + LOGGER.info('DHCP server stopped: ' + (str(stopped))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to stop DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def AddReservedLease(self, request, context): # pylint: disable=W0613 + LOGGER.info('Add reserved lease called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.add_reserved_host(request.hostname, request.hw_addr, + request.ip_addr) + dhcp_config.write_config() + LOGGER.info('Reserved lease added') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to add reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DeleteReservedLease(self, request, context): # pylint: disable=W0613 + LOGGER.info('Delete reserved lease called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.delete_reserved_host(request.hw_addr) + dhcp_config.write_config() + LOGGER.info('Reserved lease deleted') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to delete reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DisableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Disable failover called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.disable_failover() + dhcp_config.write_config() + LOGGER.info('Failover disabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to disable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def EnableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Enable failover called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.enable_failover() + dhcp_config.write_config() + LOGGER.info('Failover enabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to enable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetDHCPRange(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP configuration and return + the first range from the first subnet in the file + """ + LOGGER.info('Get DHCP range called') + try: + pool = self._get_dhcp_config()._subnets[0].pools[0] + return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetLease(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP leased address for the + provided MAC address + """ + LOGGER.info('Get lease called') + try: + lease = self.dhcp_leases.get_lease(request.hw_addr) + if lease is not None: + return pb2.Response(code=200, message=str(lease)) + else: + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def SetDHCPRange(self, request, context): # pylint: disable=W0613 + """ + Change DHCP configuration and set the + the first range from the first subnet in the configuration + """ + LOGGER.info('Set DHCP range called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.set_range(request.start, request.end, 0, 0) + dhcp_config.write_config() + LOGGER.info('DHCP range set') + return pb2.Response(code=200, message='DHCP Range Set') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to set DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetStatus(self, request, context): # pylint: disable=W0613 + """ + Return the current status of the network module + """ + dhcp_status = self._dhcp_server.is_running() + message = str({'dhcpStatus': dhcp_status}) + return pb2.Response(code=200, message=message) diff --git a/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto new file mode 100644 index 000000000..e6abda674 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +service NetworkModule { + + rpc RestartDHCPServer(RestartDHCPServerRequest) returns (Response) {}; + + rpc StartDHCPServer(StartDHCPServerRequest) returns (Response) {}; + + rpc StopDHCPServer(StopDHCPServerRequest) returns (Response) {}; + + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; + + rpc DeleteReservedLease(DeleteReservedLeaseRequest) returns (Response) {}; + + rpc DisableFailover(DisableFailoverRequest) returns (Response) {}; + + rpc EnableFailover(EnableFailoverRequest) returns (Response) {}; + + rpc GetDHCPRange(GetDHCPRangeRequest) returns (DHCPRange) {}; + + rpc GetLease(GetLeaseRequest) returns (Response) {}; + + rpc GetStatus(GetStatusRequest) returns (Response) {}; + + rpc SetDHCPRange(SetDHCPRangeRequest) returns (Response) {}; +} + +message AddReservedLeaseRequest { + string hostname = 1; + string hw_addr = 2; + string ip_addr = 3; +} + +message DeleteReservedLeaseRequest { + string hw_addr = 1; +} + +message RestartDHCPServerRequest {} + +message StartDHCPServerRequest {} + +message StopDHCPServerRequest {} + +message DisableFailoverRequest {} + +message EnableFailoverRequest {} + +message GetDHCPRangeRequest {} + +message GetLeaseRequest { + string hw_addr = 1; +} + +message GetStatusRequest {} + +message SetDHCPRangeRequest { + int32 code = 1; + string start = 2; + string end = 3; +} + +message Response { + int32 code = 1; + string message = 2; +} + +message DHCPRange { + int32 code = 1; + string start = 2; + string end = 3; +} \ No newline at end of file diff --git a/modules/network/dhcp-2/python/src/grpc_server/radvd_server.py b/modules/network/dhcp-2/python/src/grpc_server/radvd_server.py new file mode 100644 index 000000000..910354e31 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/radvd_server.py @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +RADVD server booted from the radvd-service file""" +from common import logger +from common import util + +LOG_NAME = 'radvd' +LOGGER = None + +class RADVDServer: + """Represents the RADVD Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def restart(self): + LOGGER.info('Restarting RADVD server') + response = util.run_command('radvd-service restart', False) + LOGGER.info('RADVD restarted: ' + str(response)) + return response + + def start(self): + LOGGER.info('Starting RADVD server') + response = util.run_command('radvd-service start', False) + LOGGER.info('RADVD started: ' + str(response)) + return response + + def stop(self): + LOGGER.info('Stopping RADVD server') + response = util.run_command('radvd-service stop', False) + LOGGER.info('RADVD stopped: ' + str(response)) + return response + + def is_running(self): + LOGGER.info('Checking RADVD status') + response = util.run_command('radvd-service status') + running = response[0] == 'radvd service is running.' + LOGGER.info('RADVD status: ' + str(running)) + return running diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index 9c7f2bac2..10344cbc7 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -36,5 +36,13 @@ RUN dos2unix /testrun/bin/* # Make sure all the bin files are executable RUN chmod u+x /testrun/bin/* +# Copy over all network module gRPC proto files +ARG NET_MODULE_DIR=modules/network +ARG NET_MODULE_PROTO_DIR=python/src/grpc_server/proto/grpc.proto +ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc_server/proto + +COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ +COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ + # Start the test module ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/modules/test/base/bin/setup_grpc_clients b/modules/test/base/bin/setup_grpc_clients new file mode 100644 index 000000000..30efe5002 --- /dev/null +++ b/modules/test/base/bin/setup_grpc_clients @@ -0,0 +1,34 @@ +#!/bin/bash -e + +GRPC_DIR="/testrun/python/src/grpc_server" +GRPC_PROTO_DIR="proto" +GRPC_PROTO_FILE="grpc.proto" + +# Build the grpc proto file +build_grpc_client(){ + MODULE=$1 + echo "Building gRPC proto: $MODULE" + python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$MODULE/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. +} + +# Build the grpc proto files for every module that has a proto defined +build_grpc_clients(){ + + for dir in "$GRPC_DIR/$GRPC_PROTO_DIR"/*/;do + if [ -f $dir/$GRPC_PROTO_FILE ];then + # Extract the last folder name + last_folder="${dir%%/}" + last_folder="${last_folder##*/}" + build_grpc_client "$last_folder" + fi + done +} + +# Move into the grpc directory. +# This is necessary to build the proto files +# with the correct import paths +pushd $GRPC_DIR >/dev/null 2>&1 + +build_grpc_clients + +popd >/dev/null 2>&1 \ No newline at end of file diff --git a/modules/test/base/bin/setup_python_path b/modules/test/base/bin/setup_python_path new file mode 100644 index 000000000..8201bbb36 --- /dev/null +++ b/modules/test/base/bin/setup_python_path @@ -0,0 +1,25 @@ +#!/bin/bash + +ROOT_DIRECTORY="/testrun/python/src" + +# Function to recursively add subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath() { + local directory=$1 + local subdirectories=( "$directory"/* ) + local subdirectory + + for subdirectory in "${subdirectories[@]}"; do + if [ -d "$subdirectory" ]; then + export PYTHONPATH="$PYTHONPATH:$subdirectory" + add_subdirectories_to_pythonpath "$subdirectory" + fi + done +} + +# Set PYTHONPATH initially to an empty string +export PYTHONPATH="" + +# Add all subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath "$ROOT_DIRECTORY" + +echo "$PYTHONPATH" \ No newline at end of file diff --git a/modules/test/base/bin/start_module b/modules/test/base/bin/start_module index 5f6e1ee35..82c9d26bf 100644 --- a/modules/test/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -57,10 +57,21 @@ then exit 1 fi -echo "Starting module $MODULE_NAME..." +# Setup the PYTHONPATH so all imports work as expected +echo "Setting up PYTHONPATH..." +export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" + +# Build all gRPC files from the proto for use in +# gRPC clients for communications to network modules +echo "Building gRPC files from available proto files..." +$BIN_DIR/setup_grpc_clients +echo "Configuring binary files..." $BIN_DIR/setup_binaries $BIN_DIR +echo "Starting module $MODULE_NAME..." + # Only start network services if the test container needs # a network connection to run its tests if [ $NETWORK_REQUIRED == "true" ];then @@ -78,9 +89,9 @@ then if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] then echo "gRPC port resolved from config: $GRPC_PORT" - $BIN_DIR/start_grpc "-p $GRPC_PORT" & + $BIN_DIR/start_grpc "-p $GRPC_PORT" else - $BIN_DIR/start_grpc & + $BIN_DIR/start_grpc fi fi diff --git a/modules/test/base/python/src/grpc/proto/dhcp1/client.py b/modules/test/base/python/src/grpc/proto/dhcp1/client.py new file mode 100644 index 000000000..03dd873bd --- /dev/null +++ b/modules/test/base/python/src/grpc/proto/dhcp1/client.py @@ -0,0 +1,131 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""gRPC client module for the primary DHCP Server""" + +import grpc +import grpc_pb2_grpc as pb2_grpc +import grpc_pb2 as pb2 + +DEFAULT_PORT = '5001' +DEFAULT_HOST = '10.10.10.2' # Default DHCP1 server + + +class Client(): + """gRPC Client for the primary DHCP server""" + def __init__(self, port=DEFAULT_PORT, host=DEFAULT_HOST): + self._port = port + self._host = host + + # Create a gRPC channel to connect to the server + self._channel = grpc.insecure_channel(self._host + ':' + self._port) + + # Create a gRPC stub + self._stub = pb2_grpc.NetworkModuleStub(self._channel) + + def add_reserved_lease(self, hostname, hw_addr, ip_addr): + # Create a request message + request = pb2.AddReservedLeaseRequest() + request.hostname = hostname + request.hw_addr = hw_addr + request.ip_addr = ip_addr + + # Make the RPC call + response = self._stub.AddReservedLease(request) + + return response + + def delete_reserved_lease(self, hw_addr): + # Create a request message + request = pb2.DeleteReservedLeaseRequest() + request.hw_addr = hw_addr + + # Make the RPC call + response = self._stub.DeleteReservedLease(request) + + return response + + def disable_failover(self): + # Create a request message + request = pb2.DisableFailoverRequest() + + # Make the RPC call + response = self._stub.DisableFailover(request) + + return response + + def enable_failover(self): + # Create a request message + request = pb2.EnableFailoverRequest() + + # Make the RPC call + response = self._stub.EnableFailover(request) + + return response + + def get_dhcp_range(self): + # Create a request message + request = pb2.GetDHCPRangeRequest() + + # Make the RPC call + response = self._stub.GetDHCPRange(request) + + return response + + def get_lease(self,hw_addr): + # Create a request message + request = pb2.GetLeaseRequest() + request.hw_addr=hw_addr + + # Make the RPC call + response = self._stub.GetLease(request) + + return response + + def get_status(self): + # Create a request message + request = pb2.GetStatusRequest() + + # Make the RPC call + response = self._stub.GetStatus(request) + + return response + + def stop_dhcp_server(self): + # Create a request message + request = pb2.StopDHCPServerRequest() + + # Make the RPC call + response = self._stub.StopDHCPServer(request) + + return response + + def start_dhcp_server(self): + # Create a request message + request = pb2.StartDHCPServerRequest() + + # Make the RPC call + response = self._stub.StartDHCPServer(request) + + return response + + def set_dhcp_range(self,start,end): + # Create a request message + request = pb2.SetDHCPRangeRequest() + request.start=start + request.end=end + + # Make the RPC call + response = self._stub.SetDHCPRange(request) + + return response diff --git a/modules/test/base/python/src/grpc/proto/dhcp2/client.py b/modules/test/base/python/src/grpc/proto/dhcp2/client.py new file mode 100644 index 000000000..e0d953ee5 --- /dev/null +++ b/modules/test/base/python/src/grpc/proto/dhcp2/client.py @@ -0,0 +1,130 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""gRPC client module for the secondary DHCP Server""" +import grpc +import grpc_pb2_grpc as pb2_grpc +import grpc_pb2 as pb2 + +DEFAULT_PORT = '5001' +DEFAULT_HOST = '10.10.10.3' # Default DHCP2 server + + +class Client(): + """gRPC Client for the secondary DHCP server""" + def __init__(self, port=DEFAULT_PORT, host=DEFAULT_HOST): + self._port = port + self._host = host + + # Create a gRPC channel to connect to the server + self._channel = grpc.insecure_channel(self._host + ':' + self._port) + + # Create a gRPC stub + self._stub = pb2_grpc.NetworkModuleStub(self._channel) + + def add_reserved_lease(self, hostname, hw_addr, ip_addr): + # Create a request message + request = pb2.AddReservedLeaseRequest() + request.hostname = hostname + request.hw_addr = hw_addr + request.ip_addr = ip_addr + + # Make the RPC call + response = self._stub.AddReservedLease(request) + + return response + + def delete_reserved_lease(self, hw_addr): + # Create a request message + request = pb2.DeleteReservedLeaseRequest() + request.hw_addr = hw_addr + + # Make the RPC call + response = self._stub.DeleteReservedLease(request) + + return response + + def disable_failover(self): + # Create a request message + request = pb2.DisableFailoverRequest() + + # Make the RPC call + response = self._stub.DisableFailover(request) + + return response + + def enable_failover(self): + # Create a request message + request = pb2.EnableFailoverRequest() + + # Make the RPC call + response = self._stub.EnableFailover(request) + + return response + + def get_dhcp_range(self): + # Create a request message + request = pb2.GetDHCPRangeRequest() + + # Make the RPC call + response = self._stub.GetDHCPRange(request) + + return response + + def get_lease(self,hw_addr): + # Create a request message + request = pb2.GetLeaseRequest() + request.hw_addr=hw_addr + + # Make the RPC call + response = self._stub.GetLease(request) + + return response + + def get_status(self): + # Create a request message + request = pb2.GetStatusRequest() + + # Make the RPC call + response = self._stub.GetStatus(request) + + return response + + def stop_dhcp_server(self): + # Create a request message + request = pb2.StopDHCPServerRequest() + + # Make the RPC call + response = self._stub.StopDHCPServer(request) + + return response + + def start_dhcp_server(self): + # Create a request message + request = pb2.StartDHCPServerRequest() + + # Make the RPC call + response = self._stub.StartDHCPServer(request) + + return response + + def set_dhcp_range(self,start,end): + # Create a request message + request = pb2.SetDHCPRangeRequest() + request.start=start + request.end=end + + # Make the RPC call + response = self._stub.SetDHCPRange(request) + + return response diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 5342e36f8..b0898aa20 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -58,10 +58,11 @@ def _get_device_tests(self, device_test_module): for test in module_tests: # Resolve device specific configurations for the test if it exists # and update module test config with device config options - if test['name'] in device_test_module['tests']: - dev_test_config = device_test_module['tests'][test['name']] - if 'config' in test: - test['config'].update(dev_test_config) + if 'tests' in device_test_module: + if test['name'] in device_test_module['tests']: + dev_test_config = device_test_module['tests'][test['name']] + if 'config' in test: + test['config'].update(dev_test_config) return module_tests def _get_device_test_module(self): @@ -89,14 +90,13 @@ def run_tests(self): else: result = getattr(self, test_method_name)() else: - LOGGER.info('Test ' + test['name'] + ' not resolved. Skipping') + LOGGER.info(f'Test {test["name"]} not resolved. Skipping') result = None else: - LOGGER.info('Test ' + test['name'] + ' disabled. Skipping') + LOGGER.info(f'Test {test["name"]} disabled. Skipping') if result is not None: - success = None - if isinstance(result,bool): - test['result'] = 'compliant' if result else 'non-compliant' + if isinstance(result, bool): + test['result'] = 'compliant' if result else 'non-compliant' else: test['result'] = 'compliant' if result[0] else 'non-compliant' test['result_details'] = result[1] diff --git a/modules/test/conn/conf/module_config.json b/modules/test/conn/conf/module_config.json index 0f599c5d3..b82879544 100644 --- a/modules/test/conn/conf/module_config.json +++ b/modules/test/conn/conf/module_config.json @@ -9,9 +9,14 @@ "docker": { "depends_on": "base", "enable_container": true, - "timeout": 30 + "timeout": 600 }, "tests": [ + { + "name": "connection.dhcp_address", + "description": "The device under test has received an IP address from the DHCP server and responds to an ICMP echo (ping) request", + "expected_behavior": "The device is not setup with a static IP address. The device accepts an IP address from a DHCP server (RFC 2131) and responds succesfully to an ICMP echo (ping) request." + }, { "name": "connection.mac_address", "description": "Check and note device physical address.", @@ -22,6 +27,25 @@ "description": "The device under test hs a MAC address prefix that is registered against a known manufacturer.", "expected_behavior": "The MAC address prefix is registered in the IEEE Organizationally Unique Identifier database." }, + { + "name": "connection.private_address", + "description": "The device under test accepts an IP address that is compliant with RFC 1918 Address Allocation for Private Internets.", + "expected_behavior": "The device under test accepts IP addresses within all ranges specified in RFC 1918 and communicates using these addresses. The Internet Assigned Numbers Authority (IANA) has reserved the following three blocks of the IP address space for private internets. 10.0.0.0 - 10.255.255.255.255 (10/8 prefix). 172.16.0.0 - 172.31.255.255 (172.16/12 prefix). 192.168.0.0 - 192.168.255.255 (192.168/16 prefix)", + "config": [ + { + "start": "10.0.0.100", + "end": "10.0.0.200" + }, + { + "start":"172.16.0.0", + "end":"172.16.255.255" + }, + { + "start":"192.168.0.0", + "end":"192.168.255.255" + } + ] + }, { "name": "connection.single_ip", "description": "The network switch port connected to the device reports only one IP address for the device under test.", @@ -31,6 +55,16 @@ "name": "connection.target_ping", "description": "The device under test responds to an ICMP echo (ping) request.", "expected_behavior": "The device under test responds to an ICMP echo (ping) request." + }, + { + "name": "connection.ipv6_slaac", + "description": "The device forms a valid IPv6 address as a combination of the IPv6 router prefix and the device interface identifier", + "expected_behavior": "The device under test complies with RFC4862 and forms a valid IPv6 SLAAC address" + }, + { + "name": "connection.ipv6_ping", + "description": "The device responds to an IPv6 ping (ICMPv6 Echo) request to the SLAAC address", + "expected_behavior": "The device responds to the ping as per RFC4443" } ] } diff --git a/modules/test/conn/conn.Dockerfile b/modules/test/conn/conn.Dockerfile index 1714f49f2..5d8148335 100644 --- a/modules/test/conn/conn.Dockerfile +++ b/modules/test/conn/conn.Dockerfile @@ -17,6 +17,8 @@ FROM test-run/base-test:latest ARG MODULE_NAME=conn ARG MODULE_DIR=modules/test/$MODULE_NAME +ARG GRPC_PROTO_DIR=/testrun/python/src/grpc/proto/dhcp +ARG GRPC_PROTO_FILE="grpc.proto" # Install all necessary packages RUN apt-get install -y wget @@ -37,4 +39,4 @@ COPY $MODULE_DIR/conf /testrun/conf COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY $MODULE_DIR/python /testrun/python +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index 196c335d8..da8754608 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -11,19 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Connection test module""" import util import sys -from scapy.all import * +import time +from datetime import datetime +from scapy.all import rdpcap, DHCP, Ether, IPv6, ICMPv6ND_NS from test_module import TestModule +from dhcp1.client import Client as DHCPClient1 +from dhcp2.client import Client as DHCPClient2 -LOG_NAME = "test_connection" +LOG_NAME = 'test_connection' LOGGER = None -OUI_FILE="/usr/local/etc/oui.txt" +OUI_FILE = '/usr/local/etc/oui.txt' DHCP_SERVER_CAPTURE_FILE = '/runtime/network/dhcp-1.pcap' STARTUP_CAPTURE_FILE = '/runtime/device/startup.pcap' MONITOR_CAPTURE_FILE = '/runtime/device/monitor.pcap' +SLAAC_PREFIX = 'fd10:77be:4186' class ConnectionModule(TestModule): @@ -33,34 +37,150 @@ def __init__(self, module): super().__init__(module_name=module, log_name=LOG_NAME) global LOGGER LOGGER = self._get_logger() + self.dhcp1_client = DHCPClient1() + self.dhcp2_client = DHCPClient2() + + # ToDo: Move this into some level of testing, leave for + # reference until tests are implemented with these calls + # response = self.dhcp1_client.add_reserved_lease( + # 'test','00:11:22:33:44:55','10.10.10.21') + # print("AddLeaseResp: " + str(response)) + + # response = self.dhcp1_client.delete_reserved_lease('00:11:22:33:44:55') + # print("DelLeaseResp: " + str(response)) + + # response = self.dhcp1_client.disable_failover() + # print("FailoverDisabled: " + str(response)) + + # response = self.dhcp1_client.enable_failover() + # print("FailoverEnabled: " + str(response)) + + # response = self.dhcp1_client.get_dhcp_range() + # print("DHCP Range: " + str(response)) + + # response = self.dhcp1_client.get_lease(self._device_mac) + # print("Lease: " + str(response)) + + # response = self.dhcp1_client.get_status() + # print("Status: " + str(response)) + + # response = self.dhcp1_client.set_dhcp_range('10.10.10.20','10.10.10.30') + # print("Set Range: " + str(response)) + + def _connection_private_address(self, config): + # Shutdown the secondary DHCP Server + LOGGER.info('Running connection.private_address') + response = self.dhcp1_client.get_dhcp_range() + cur_range = {} + if response.code == 200: + cur_range['start'] = response.start + cur_range['end'] = response.end + LOGGER.info('Current DHCP subnet range: ' + str(cur_range)) + else: + LOGGER.error('Failed to resolve current subnet range required ' + 'for restoring network') + return None, ('Failed to resolve current subnet range required ' + 'for restoring network') + + results = [] + dhcp_setup = self.setup_single_dhcp_server() + if dhcp_setup[0]: + LOGGER.info(dhcp_setup[1]) + lease = self._get_cur_lease() + if lease is not None: + if self._is_lease_active(lease): + results = self.test_subnets(config) + else: + return None, 'Failed to confirm a valid active lease for the device' + else: + LOGGER.error(dhcp_setup[1]) + return None, 'Failed to setup DHCP server for test' + + # Process and return final results + final_result = None + final_result_details = '' + for result in results: + if final_result is None: + final_result = result['result'] + else: + final_result &= result['result'] + final_result_details += result['details'] + '\n' + + try: + # Restore failover configuration of DHCP servers + self.restore_failover_dhcp_server(cur_range) + + # Wait for the current lease to expire + self._wait_for_lease_expire(self._get_cur_lease()) + + # Wait for a new lease to be provided before exiting test + # to prevent other test modules from failing + for _ in range(5): + LOGGER.info('Checking for new lease') + lease = self._get_cur_lease() + if lease is not None: + LOGGER.info('New Lease found: ' + str(lease)) + LOGGER.info('Validating subnet for new lease...') + in_range = self.is_ip_in_range(lease['ip'], cur_range['start'], + cur_range['end']) + LOGGER.info('Lease within subnet: ' + str(in_range)) + break + else: + LOGGER.info('New lease not found. Waiting to check again') + time.sleep(5) + + except Exception as e: # pylint: disable=W0718 + LOGGER.error('Failed to restore DHCP server configuration: ' + str(e)) + + return final_result, final_result_details + + def _connection_dhcp_address(self): + LOGGER.info('Running connection.dhcp_address') + response = self.dhcp1_client.get_lease(self._device_mac) + LOGGER.info('DHCP Lease resolved:\n' + str(response)) + if response.code == 200: + lease = eval(response.message) # pylint: disable=W0123 + if 'ip' in lease: + ip_addr = lease['ip'] + LOGGER.info('IP Resolved: ' + ip_addr) + LOGGER.info('Attempting to ping device...') + ping_success = self._ping(self._device_ipv4_addr) + LOGGER.info('Ping Success: ' + str(ping_success)) + if ping_success: + return True, 'Device responded to leased ip address' + else: + return False, 'Device did not respond to leased ip address' + else: + LOGGER.info('No DHCP lease found for: ' + self._device_mac) + return False, 'No DHCP lease found for: ' + self._device_mac def _connection_mac_address(self): - LOGGER.info("Running connection.mac_address") + LOGGER.info('Running connection.mac_address') if self._device_mac is not None: - LOGGER.info("MAC address found: " + self._device_mac) - return True, "MAC address found: " + self._device_mac + LOGGER.info('MAC address found: ' + self._device_mac) + return True, 'MAC address found: ' + self._device_mac else: - LOGGER.info("No MAC address found: " + self._device_mac) - return False, "No MAC address found." + LOGGER.info('No MAC address found: ' + self._device_mac) + return False, 'No MAC address found.' def _connection_mac_oui(self): - LOGGER.info("Running connection.mac_oui") + LOGGER.info('Running connection.mac_oui') manufacturer = self._get_oui_manufacturer(self._device_mac) if manufacturer is not None: - LOGGER.info("OUI Manufacturer found: " + manufacturer) - return True, "OUI Manufacturer found: " + manufacturer + LOGGER.info('OUI Manufacturer found: ' + manufacturer) + return True, 'OUI Manufacturer found: ' + manufacturer else: - LOGGER.info("No OUI Manufacturer found for: " + self._device_mac) - return False, "No OUI Manufacturer found for: " + self._device_mac + LOGGER.info('No OUI Manufacturer found for: ' + self._device_mac) + return False, 'No OUI Manufacturer found for: ' + self._device_mac def _connection_single_ip(self): - LOGGER.info("Running connection.single_ip") + LOGGER.info('Running connection.single_ip') result = None if self._device_mac is None: - LOGGER.info("No MAC address found: ") - return result, "No MAC address found." - + LOGGER.info('No MAC address found: ') + return result, 'No MAC address found.' + # Read all the pcap files containing DHCP packet information packets = rdpcap(DHCP_SERVER_CAPTURE_FILE) packets.append(rdpcap(STARTUP_CAPTURE_FILE)) @@ -68,50 +188,244 @@ def _connection_single_ip(self): # Extract MAC addresses from DHCP packets mac_addresses = set() - LOGGER.info("Inspecting: " + str(len(packets)) + " packets") + LOGGER.info('Inspecting: ' + str(len(packets)) + ' packets') for packet in packets: # Option[1] = message-type, option 3 = DHCPREQUEST - if DHCP in packet and packet[DHCP].options[0][1] == 3: - mac_address = packet[Ether].src - mac_addresses.add(mac_address.upper()) + if DHCP in packet and packet[DHCP].options[0][1] == 3: + mac_address = packet[Ether].src + mac_addresses.add(mac_address.upper()) # Check if the device mac address is in the list of DHCPREQUESTs result = self._device_mac.upper() in mac_addresses - LOGGER.info("DHCPREQUEST detected from device: " + str(result)) + LOGGER.info('DHCPREQUEST detected from device: ' + str(result)) # Check the unique MAC addresses to see if they match the device for mac_address in mac_addresses: - LOGGER.info("DHCPREQUEST from MAC address: " + mac_address) - result &= self._device_mac.upper() == mac_address + LOGGER.info('DHCPREQUEST from MAC address: ' + mac_address) + result &= self._device_mac.upper() == mac_address return result - def _connection_target_ping(self): - LOGGER.info("Running connection.target_ping") + LOGGER.info('Running connection.target_ping') # If the ipv4 address wasn't resolved yet, try again if self._device_ipv4_addr is None: - self._device_ipv4_addr = self._get_device_ipv4(self) + self._device_ipv4_addr = self._get_device_ipv4(self) if self._device_ipv4_addr is None: - LOGGER.error("No device IP could be resolved") + LOGGER.error('No device IP could be resolved') sys.exit(1) else: return self._ping(self._device_ipv4_addr) - def _get_oui_manufacturer(self,mac_address): + def _get_oui_manufacturer(self, mac_address): # Do some quick fixes on the format of the mac_address # to match the oui file pattern - mac_address = mac_address.replace(":","-").upper() - with open(OUI_FILE, "r") as file: - for line in file: - if mac_address.startswith(line[:8]): - start = line.index("(hex)") + len("(hex)") - return line[start:].strip() # Extract the company name + mac_address = mac_address.replace(':', '-').upper() + with open(OUI_FILE, 'r', encoding='UTF-8') as file: + for line in file: + if mac_address.startswith(line[:8]): + start = line.index('(hex)') + len('(hex)') + return line[start:].strip() # Extract the company name return None + def _connection_ipv6_slaac(self): + LOGGER.info('Running connection.ipv6_slaac') + packet_capture = rdpcap(MONITOR_CAPTURE_FILE) + + sends_ipv6 = False + + for packet in packet_capture: + if IPv6 in packet and packet.src == self._device_mac: + sends_ipv6 = True + if ICMPv6ND_NS in packet: + ipv6_addr = str(packet[ICMPv6ND_NS].tgt) + if ipv6_addr.startswith(SLAAC_PREFIX): + self._device_ipv6_addr = ipv6_addr + LOGGER.info(f'Device has formed SLAAC address {ipv6_addr}') + return True + + if sends_ipv6: + LOGGER.info('Device does not support IPv6 SLAAC') + else: + LOGGER.info('Device does not support IPv6') + return False + + def _connection_ipv6_ping(self): + LOGGER.info('Running connection.ipv6_ping') + + if self._device_ipv6_addr is None: + LOGGER.info('No IPv6 SLAAC address found. Cannot ping') + return + + if self._ping(self._device_ipv6_addr): + LOGGER.info(f'Device responds to IPv6 ping on {self._device_ipv6_addr}') + return True + else: + LOGGER.info('Device does not respond to IPv6 ping') + return False + def _ping(self, host): - cmd = 'ping -c 1 ' + str(host) + cmd = "ping -c 1 " + str(host) success = util.run_command(cmd, output=False) return success - \ No newline at end of file + + def restore_failover_dhcp_server(self, subnet): + # Configure the subnet range + if self._change_subnet(subnet): + if self.enable_failover(): + response = self.dhcp2_client.start_dhcp_server() + if response.code == 200: + LOGGER.info('DHCP server configuration restored') + return True + else: + LOGGER.error('Failed to start secondary DHCP server') + return False + else: + LOGGER.error('Failed to enabled failover in primary DHCP server') + return False + else: + LOGGER.error('Failed to restore original subnet') + return False + + def setup_single_dhcp_server(self): + # Shutdown the secondary DHCP Server + LOGGER.info('Stopping secondary DHCP server') + response = self.dhcp2_client.stop_dhcp_server() + if response.code == 200: + LOGGER.info('Secondary DHCP server stop command success') + time.sleep(3) # Give some time for the server to stop + LOGGER.info('Checking secondary DHCP server status') + response = self.dhcp2_client.get_status() + if response.code == 200: + LOGGER.info('Secondary DHCP server stopped') + return True, 'Single DHCP server configured' + else: + return False, 'DHCP server still running' + else: + return False, 'DHCP server stop command failed' + + # Move primary DHCP server from failover into a single DHCP server config + LOGGER.info('Configuring primary DHCP server') + response = self.dhcp1_client.disable_failover() + if response.code == 200: + LOGGER.info('Primary DHCP server failover disabled') + else: + return False, 'Failed to disable primary DHCP server failover' + + def enable_failover(self): + # Move primary DHCP server to primary failover + LOGGER.info('Configuring primary failover DHCP server') + response = self.dhcp1_client.enable_failover() + if response.code == 200: + LOGGER.info('Primary DHCP server enabled') + return True + else: + LOGGER.error('Failed to disable primary DHCP server failover') + return False + + def is_ip_in_range(self, ip, start_ip, end_ip): + ip_int = int(''.join(format(int(octet), '08b') for octet in ip.split('.')), + 2) + start_int = int( + ''.join(format(int(octet), '08b') for octet in start_ip.split('.')), 2) + end_int = int( + ''.join(format(int(octet), '08b') for octet in end_ip.split('.')), 2) + + return start_int <= ip_int <= end_int + + def _test_subnet(self, subnet, lease): + if self._change_subnet(subnet): + expiration = datetime.strptime(lease['expires'], '%Y-%m-%d %H:%M:%S') + time_to_expire = expiration - datetime.now() + LOGGER.info('Time until lease expiration: ' + str(time_to_expire)) + LOGGER.info('Waiting for current lease to expire: ' + str(expiration)) + if time_to_expire.total_seconds() > 0: + time.sleep(time_to_expire.total_seconds() + + 5) # Wait until the expiration time and padd 5 seconds + LOGGER.info('Current lease expired. Checking for new lease') + for _ in range(5): + LOGGER.info('Checking for new lease') + lease = self._get_cur_lease() + if lease is not None: + LOGGER.info('New Lease found: ' + str(lease)) + LOGGER.info('Validating subnet for new lease...') + in_range = self.is_ip_in_range(lease['ip'], subnet['start'], + subnet['end']) + LOGGER.info('Lease within subnet: ' + str(in_range)) + return in_range + else: + LOGGER.info('New lease not found. Waiting to check again') + time.sleep(5) + + def _wait_for_lease_expire(self, lease): + expiration = datetime.strptime(lease['expires'], '%Y-%m-%d %H:%M:%S') + time_to_expire = expiration - datetime.now() + LOGGER.info('Time until lease expiration: ' + str(time_to_expire)) + LOGGER.info('Waiting for current lease to expire: ' + str(expiration)) + if time_to_expire.total_seconds() > 0: + time.sleep(time_to_expire.total_seconds() + + 5) # Wait until the expiration time and padd 5 seconds + LOGGER.info('Current lease expired.') + + def _change_subnet(self, subnet): + LOGGER.info('Changing subnet to: ' + str(subnet)) + response = self.dhcp1_client.set_dhcp_range(subnet['start'], subnet['end']) + if response.code == 200: + LOGGER.info('Subnet change request accepted. Confirming change...') + response = self.dhcp1_client.get_dhcp_range() + if response.code == 200: + if response.start == subnet['start'] and response.end == subnet['end']: + LOGGER.info('Subnet change confirmed') + return True + LOGGER.error('Failed to confirm subnet change') + else: + LOGGER.error('Subnet change request failed.') + return False + + def _get_cur_lease(self): + LOGGER.info('Checking current device lease') + response = self.dhcp1_client.get_lease(self._device_mac) + if response.code == 200: + lease = eval(response.message) # pylint: disable=W0123 + if lease: # Check if non-empty lease + return lease + else: + return None + + def _is_lease_active(self, lease): + if 'ip' in lease: + ip_addr = lease['ip'] + LOGGER.info('Lease IP Resolved: ' + ip_addr) + LOGGER.info('Attempting to ping device...') + ping_success = self._ping(self._device_ipv4_addr) + LOGGER.info('Ping Success: ' + str(ping_success)) + LOGGER.info('Current lease confirmed active in device') + return ping_success + + def test_subnets(self, subnets): + results = [] + for subnet in subnets: + result = {} + try: + lease = self._get_cur_lease() + if lease is not None: + result = self._test_subnet(subnet, lease) + if result: + result = { + 'result': + True, + 'details': + 'Subnet ' + subnet['start'] + '-' + subnet['end'] + ' passed' + } + else: + result = { + 'result': + False, + 'details': + 'Subnet ' + subnet['start'] + '-' + subnet['end'] + ' failed' + } + except Exception as e: # pylint: disable=W0718 + result = {'result': False, 'details': 'Subnet test failed: ' + str(e)} + results.append(result) + return results diff --git a/modules/test/nmap/nmap.Dockerfile b/modules/test/nmap/nmap.Dockerfile index 1789da382..ea90ee06f 100644 --- a/modules/test/nmap/nmap.Dockerfile +++ b/modules/test/nmap/nmap.Dockerfile @@ -18,10 +18,10 @@ FROM test-run/base-test:latest ARG MODULE_NAME=nmap ARG MODULE_DIR=modules/test/$MODULE_NAME -#Load the requirements file +# Load the requirements file COPY $MODULE_DIR/python/requirements.txt /testrun/python -#Install all python requirements for the module +# Install all python requirements for the module RUN pip3 install -r /testrun/python/requirements.txt # Copy over all configuration files diff --git a/modules/test/ntp/bin/start_test_module b/modules/test/ntp/bin/start_test_module new file mode 100644 index 000000000..a09349cf9 --- /dev/null +++ b/modules/test/ntp/bin/start_test_module @@ -0,0 +1,42 @@ +#!/bin/bash + +# An example startup script that does the bare minimum to start +# a test module via a pyhon script. Each test module should include a +# start_test_module file that overwrites this one to boot all of its +# specific requirements to run. + +# Define where the python source files are located +PYTHON_SRC_DIR=/testrun/python/src + +# Fetch module name +MODULE_NAME=$1 + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Allow a user to define an interface by passing it into this script +DEFINED_IFACE=$2 + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No interface defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +# Create and set permissions on the log files +LOG_FILE=/runtime/output/$MODULE_NAME.log +RESULT_FILE=/runtime/output/$MODULE_NAME-result.json +touch $LOG_FILE +touch $RESULT_FILE +chown $HOST_USER $LOG_FILE +chown $HOST_USER $RESULT_FILE + +# Run the python scrip that will execute the tests for this module +# -u flag allows python print statements +# to be logged by docker by running unbuffered +python3 -u $PYTHON_SRC_DIR/run.py "-m $MODULE_NAME" + +echo Module has finished \ No newline at end of file diff --git a/modules/test/ntp/conf/module_config.json b/modules/test/ntp/conf/module_config.json new file mode 100644 index 000000000..288474868 --- /dev/null +++ b/modules/test/ntp/conf/module_config.json @@ -0,0 +1,27 @@ +{ + "config": { + "meta": { + "name": "ntp", + "display_name": "NTP", + "description": "NTP test" + }, + "network": false, + "docker": { + "depends_on": "base", + "enable_container": true, + "timeout": 30 + }, + "tests":[ + { + "name": "ntp.network.ntp_support", + "description": "Does the device request network time sync as client as per RFC 5905 - Network Time Protocol Version 4: Protocol and Algorithms Specification", + "expected_behavior": "The device sends an NTPv4 request to the configured NTP server." + }, + { + "name": "ntp.network.ntp_dhcp", + "description": "Accept NTP address over DHCP", + "expected_behavior": "Device can accept NTP server address, provided by the DHCP server (DHCP OFFER PACKET)" + } + ] + } +} diff --git a/modules/test/ntp/ntp.Dockerfile b/modules/test/ntp/ntp.Dockerfile new file mode 100644 index 000000000..33b06287e --- /dev/null +++ b/modules/test/ntp/ntp.Dockerfile @@ -0,0 +1,20 @@ +# Image name: test-run/ntp-test +FROM test-run/base-test:latest + +ARG MODULE_NAME=ntp +ARG MODULE_DIR=modules/test/$MODULE_NAME + +# Load the requirements file +COPY $MODULE_DIR/python/requirements.txt /testrun/python + +# Install all python requirements for the module +RUN pip3 install -r /testrun/python/requirements.txt + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/test/ntp/python/requirements.txt b/modules/test/ntp/python/requirements.txt new file mode 100644 index 000000000..93b351f44 --- /dev/null +++ b/modules/test/ntp/python/requirements.txt @@ -0,0 +1 @@ +scapy \ No newline at end of file diff --git a/modules/test/ntp/python/src/ntp_module.py b/modules/test/ntp/python/src/ntp_module.py new file mode 100644 index 000000000..4053ce98a --- /dev/null +++ b/modules/test/ntp/python/src/ntp_module.py @@ -0,0 +1,79 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NTP test module""" +from test_module import TestModule +from scapy.all import rdpcap, NTP, IP + +LOG_NAME = 'test_ntp' +NTP_SERVER_CAPTURE_FILE = '/runtime/network/ntp.pcap' +STARTUP_CAPTURE_FILE = '/runtime/device/startup.pcap' +MONITOR_CAPTURE_FILE = '/runtime/device/monitor.pcap' +LOGGER = None + +class NTPModule(TestModule): + """NTP Test module""" + + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + # TODO: This should be fetched dynamically + self._ntp_server = '10.10.10.5' + + global LOGGER + LOGGER = self._get_logger() + + def _ntp_network_ntp_support(self): + LOGGER.info('Running ntp.network.ntp_support') + + packet_capture = rdpcap(STARTUP_CAPTURE_FILE) + rdpcap(MONITOR_CAPTURE_FILE) + + device_sends_ntp4 = False + device_sends_ntp3 = False + + for packet in packet_capture: + + if NTP in packet and packet.src == self._device_mac: + if packet[NTP].version == 4: + device_sends_ntp4 = True + LOGGER.info(f'Device sent NTPv4 request to {packet[IP].dst}') + elif packet[NTP].version == 3: + device_sends_ntp3 = True + LOGGER.info(f'Device sent NTPv3 request to {packet[IP].dst}') + + if not (device_sends_ntp3 or device_sends_ntp4): + LOGGER.info('Device has not sent any NTP requests') + + return device_sends_ntp4 and not device_sends_ntp3 + + def _ntp_network_ntp_dhcp(self): + LOGGER.info('Running ntp.network.ntp_dhcp') + + packet_capture = rdpcap(STARTUP_CAPTURE_FILE) + rdpcap(MONITOR_CAPTURE_FILE) + + device_sends_ntp = False + + for packet in packet_capture: + + if NTP in packet and packet.src == self._device_mac: + device_sends_ntp = True + if packet[IP].dst == self._ntp_server: + LOGGER.info('Device sent NTP request to DHCP provided NTP server') + return True + + if not device_sends_ntp: + LOGGER.info('Device has not sent any NTP requests') + else: + LOGGER.info('Device has not sent NTP requests to DHCP provided NTP server') + + return False diff --git a/modules/test/ntp/python/src/run.py b/modules/test/ntp/python/src/run.py new file mode 100644 index 000000000..685bb4083 --- /dev/null +++ b/modules/test/ntp/python/src/run.py @@ -0,0 +1,75 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run NTP test module""" +import argparse +import signal +import sys +import logger + +from ntp_module import NTPModule + +LOG_NAME = "ntp_runner" +LOGGER = logger.get_logger(LOG_NAME) + + +class NTPModuleRunner: + """Run the NTP module tests.""" + def __init__(self, module): + + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) + self.add_logger(module) + + LOGGER.info("Starting NTP test module") + + self._test_module = NTPModule(module) + self._test_module.run_tests() + + LOGGER.info("NTP test module finished") + + def add_logger(self, module): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, module) + + def _handler(self, signum): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping test module...") + LOGGER.info("Test module stopped") + sys.exit(1) + + +def run(): + parser = argparse.ArgumentParser( + description="NTP Module Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument( + "-m", + "--module", + help="Define the module name to be used to create the log file") + + args = parser.parse_args() + + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + NTPModuleRunner(args.module.strip()) + + +if __name__ == "__main__": + run() diff --git a/resources/devices/template/device_config.json b/resources/devices/template/device_config.json index 3bb804b22..1e92de25d 100644 --- a/resources/devices/template/device_config.json +++ b/resources/devices/template/device_config.json @@ -14,6 +14,35 @@ } } }, + "connection": { + "enabled": true, + "tests": { + "connection.mac_address": { + "enabled": true + }, + "connection.mac_oui": { + "enabled": true + }, + "connection.target_ping": { + "enabled": true + } + , + "connection.single_ip": { + "enabled": true + } + } + }, + "ntp": { + "enabled": true, + "tests": { + "ntp.network.ntp_support": { + "enabled": true + }, + "ntp.network.ntp_dhcp": { + "enabled": true + } + } + }, "baseline": { "enabled": false, "tests": { diff --git a/testing/docker/ci_test_device1/Dockerfile b/testing/docker/ci_test_device1/Dockerfile new file mode 100644 index 000000000..0bb697509 --- /dev/null +++ b/testing/docker/ci_test_device1/Dockerfile @@ -0,0 +1,11 @@ + +FROM ubuntu:jammy + +#Update and get all additional requirements not contained in the base image +RUN apt-get update && apt-get -y upgrade + +RUN apt-get update && apt-get install -y isc-dhcp-client ntpdate coreutils moreutils inetutils-ping curl jq dnsutils openssl netcat-openbsd + +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/testing/docker/ci_test_device1/entrypoint.sh b/testing/docker/ci_test_device1/entrypoint.sh new file mode 100755 index 000000000..8113704be --- /dev/null +++ b/testing/docker/ci_test_device1/entrypoint.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +ip a + +declare -A options +for option in $*; do + if [[ $option == *"="* ]]; then + k=$(echo $option | cut -d'=' -f1) + v=$(echo $option | cut -d'=' -f2) + options[$k]=$v + else + options[$option]=$option + fi +done + +OUT=/out/testrun_ci.json + +NTP_SERVER=10.10.10.5 +DNS_SERVER=10.10.10.4 + +function wout(){ + temp=${1//./\".\"} + key=${temp:1}\" + echo $key + value=$2 + jq "$key+=\"$value\"" $OUT | sponge $OUT +} + + +dig @8.8.8.8 +short www.google.com + +# DHCP +ip addr flush dev eth0 +PID_FILE=/var/run/dhclient.pid +if [ -f $PID_FILE ]; then + kill -9 $(cat $PID_FILE) || true + rm -f $PID_FILE +fi +dhclient -v eth0 + + +if [ -n "${options[oddservices]}" ]; then + echo Running services on non standard ports and open default ports + + echo Starting FTP 21514 and open default 20,21 + nc -nvlt -p 20 & + nc -nvlt -p 21 & + (while true; do echo -e "220 ProFTPD 1.3.5e Server (Debian) $(hostname)" | nc -l -w 1 21514; done) & + + echo Starting SMTP 1256 and open default 25, 465, 587 + nc -nvlt -p 25 & + nc -nvlt -p 465 & + nc -nvlt -p 587 & + (while true; do echo -e "220 $(hostname) ESMTP Postfix (Ubuntu)" | nc -l -w 1 1256; done) & + + echo Starting IMAP 5361 and open default ports 143, 993 + nc -nvlt -p 143 & + nc -nvlt -p 993 & + (while true; do echo -e "* OK [CAPABILITY IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE STARTTLS AUTH=PLAIN] Dovecot (Ubuntu) ready.\r\n" \ + | nc -l -w 1 5361; done) & + + echo Starting POP3 23451 and open default 110, 995 + nc -nvlt -p 110 & + nc -nvlt -p 995 & + (while true; do echo -ne "+OK POP3 Server ready\r\n" | nc -l -w 1 23451; done) & + + echo starting TFTP UDP 69 + (while true; do echo -ne "\0\x05\0\0\x07\0" | nc -u -l -w 1 69; done) & + +fi + +if [ -n "${options[snmp]}" ]; then + echo starting mock none snmpv3 on port UDP 161 + (while true; do echo -ne " \x02\x01\ " | nc -u -l -w 1 161; done) & +fi + +if [ -n "${options[snmpv3]}" ]; then + echo starting mock SNMPv3 UDP 161 + (while true; do echo -ne " \x02\x01\x030 \x02\x02Ji\x02 \x04\x01 \x02\x01\x03\x04" | nc -u -l -w 1 161; done) & +fi + +if [ -n "${options[ssh]}" ]; then + echo Starting SSH server + /usr/local/sbin/sshd +elif [ -n "${options[sshv1]}" ]; then + echo Starting SSHv1 server + echo 'Protocol 1' >> /usr/local/etc/sshd_config + /usr/local/sbin/sshd +fi + +tail -f /dev/null \ No newline at end of file diff --git a/testing/example/mac b/testing/example/mac new file mode 100644 index 000000000..e69de29bb diff --git a/testing/example/mac1/results.json b/testing/example/mac1/results.json new file mode 100644 index 000000000..e1b837225 --- /dev/null +++ b/testing/example/mac1/results.json @@ -0,0 +1,252 @@ +{ + "device": { + "mac_addr": "7e:41:12:d2:35:6a" + }, + "dns": { + "results": [ + { + "name": "dns.network.from_device", + "description": "Verify the device sends DNS requests", + "expected_behavior": "The device sends DNS requests.", + "start": "2023-07-03T13:35:48.990574", + "result": "compliant", + "end": "2023-07-03T13:35:49.035528", + "duration": "0:00:00.044954" + }, + { + "name": "dns.network.from_dhcp", + "description": "Verify the device allows for a DNS server to be entered automatically", + "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server", + "start": "2023-07-03T13:35:49.035701", + "result": "non-compliant", + "end": "2023-07-03T13:35:49.041532", + "duration": "0:00:00.005831" + }, + { + "name": "dns.mdns", + "description": "If the device has MDNS (or any kind of IP multicast), can it be disabled", + "start": "2023-07-03T13:35:49.041679", + "result": "non-compliant", + "end": "2023-07-03T13:35:49.057430", + "duration": "0:00:00.015751" + } + ] + }, + "nmap": { + "results": [ + { + "name": "security.nmap.ports", + "description": "Run an nmap scan of open ports", + "expected_behavior": "Report all open ports", + "config": { + "security.services.ftp": { + "tcp_ports": { + "20": { + "allowed": false, + "description": "File Transfer Protocol (FTP) Server Data Transfer", + "result": "compliant" + }, + "21": { + "allowed": false, + "description": "File Transfer Protocol (FTP) Server Data Transfer", + "result": "compliant" + } + }, + "description": "Check FTP port 20/21 is disabled and FTP is not running on any port", + "expected_behavior": "There is no FTP service running on any port" + }, + "security.services.ssh": { + "tcp_ports": { + "22": { + "allowed": true, + "description": "Secure Shell (SSH) server", + "version": "2.0", + "result": "compliant" + } + }, + "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", + "expected_behavior": "There is no FTP service running on any port" + }, + "security.services.telnet": { + "tcp_ports": { + "23": { + "allowed": false, + "description": "Telnet Server", + "result": "compliant" + } + }, + "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", + "expected_behavior": "There is no FTP service running on any port" + }, + "security.services.smtp": { + "tcp_ports": { + "25": { + "allowed": false, + "description": "Simple Mail Transfer Protocol (SMTP) Server", + "result": "compliant" + }, + "465": { + "allowed": false, + "description": "Simple Mail Transfer Protocol over SSL (SMTPS) Server", + "result": "compliant" + }, + "587": { + "allowed": false, + "description": "Simple Mail Transfer Protocol via TLS (SMTPS) Server", + "result": "compliant" + } + }, + "description": "Check SMTP port 25 is disabled and ports 465 or 587 with SSL encryption are (not?) enabled and SMTP is not running on any port.", + "expected_behavior": "There is no smtp service running on any port" + }, + "security.services.http": { + "tcp_ports": { + "80": { + "service_scan": { + "script": "http-methods" + }, + "allowed": false, + "description": "Administrative Insecure Web-Server", + "result": "compliant" + } + }, + "description": "Check that there is no HTTP server running on any port", + "expected_behavior": "Device is unreachable on port 80 (or any other port) and only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)" + }, + "security.services.pop": { + "tcp_ports": { + "110": { + "allowed": false, + "description": "Post Office Protocol v3 (POP3) Server", + "result": "compliant" + } + }, + "description": "Check POP port 110 is disalbed and POP is not running on any port", + "expected_behavior": "There is no pop service running on any port" + }, + "security.services.imap": { + "tcp_ports": { + "143": { + "allowed": false, + "description": "Internet Message Access Protocol (IMAP) Server", + "result": "compliant" + } + }, + "description": "Check IMAP port 143 is disabled and IMAP is not running on any port", + "expected_behavior": "There is no imap service running on any port" + }, + "security.services.snmpv3": { + "tcp_ports": { + "161": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP)", + "result": "compliant" + }, + "162": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP) Trap", + "result": "compliant" + } + }, + "udp_ports": { + "161": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP)" + }, + "162": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP) Trap" + } + }, + "description": "Check SNMP port 161/162 is disabled. If SNMP is an essential service, check it supports version 3", + "expected_behavior": "Device is unreachable on port 161 (or any other port) and device is unreachable on port 162 (or any other port) unless SNMP is essential in which case it is SNMPv3 is used." + }, + "security.services.https": { + "tcp_ports": { + "80": { + "allowed": false, + "description": "Administrative Secure Web-Server", + "result": "compliant" + } + }, + "description": "Check that if there is a web server running it is running on a secure port.", + "expected_behavior": "Device only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)" + }, + "security.services.vnc": { + "tcp_ports": { + "5800": { + "allowed": false, + "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol Over HTTP", + "result": "compliant" + }, + "5500": { + "allowed": false, + "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol", + "result": "compliant" + } + }, + "description": "Check VNC is disabled on any port", + "expected_behavior": "Device cannot be accessed /connected to via VNc on any port" + }, + "security.services.tftp": { + "udp_ports": { + "69": { + "allowed": false, + "description": "Trivial File Transfer Protocol (TFTP) Server", + "result": "compliant" + } + }, + "description": "Check TFTP port 69 is disabled (UDP)", + "expected_behavior": "There is no tftp service running on any port" + }, + "security.services.ntp": { + "udp_ports": { + "123": { + "allowed": false, + "description": "Network Time Protocol (NTP) Server", + "result": "compliant" + } + }, + "description": "Check NTP port 123 is disabled and the device is not operating as an NTP server", + "expected_behavior": "The device dos not respond to NTP requests when it's IP is set as the NTP server on another device" + } + }, + "start": "2023-07-03T13:36:26.923704", + "result": "compliant", + "end": "2023-07-03T13:36:52.965535", + "duration": "0:00:26.041831" + } + ] + }, + "baseline": { + "results": [ + { + "name": "baseline.pass", + "description": "Simulate a compliant test", + "expected_behavior": "A compliant test result is generated", + "start": "2023-07-03T13:37:29.100681", + "result": "compliant", + "end": "2023-07-03T13:37:29.100869", + "duration": "0:00:00.000188" + }, + { + "name": "baseline.fail", + "description": "Simulate a non-compliant test", + "expected_behavior": "A non-compliant test result is generated", + "start": "2023-07-03T13:37:29.100961", + "result": "non-compliant", + "end": "2023-07-03T13:37:29.101089", + "duration": "0:00:00.000128" + }, + { + "name": "baseline.skip", + "description": "Simulate a skipped test", + "expected_behavior": "A skipped test result is generated", + "start": "2023-07-03T13:37:29.101164", + "result": "skipped", + "end": "2023-07-03T13:37:29.101283", + "duration": "0:00:00.000119" + } + ] + } + } \ No newline at end of file diff --git a/testing/test_baseline b/testing/test_baseline index 64c39e397..9aec17374 100755 --- a/testing/test_baseline +++ b/testing/test_baseline @@ -20,7 +20,7 @@ ifconfig # Setup requirements sudo apt-get update -sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils +sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils isc-dhcp-client pip3 install pytest @@ -80,6 +80,6 @@ echo "Done baseline test" more $TESTRUN_OUT -pytest testing/ +pytest testing/test_baseline.py -exit $? +exit $? \ No newline at end of file diff --git a/testing/test_baseline.py b/testing/test_baseline.py index 246857581..520f909f7 100644 --- a/testing/test_baseline.py +++ b/testing/test_baseline.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" Test assertions for CI network baseline test """ +# Temporarily disabled because using Pytest fixtures +# TODO refactor fixtures to not trigger error +# pylint: disable=redefined-outer-name + import json import pytest import re @@ -24,14 +29,13 @@ @pytest.fixture def container_data(): - dir = os.path.dirname(os.path.abspath(__file__)) with open(CI_BASELINE_OUT, encoding='utf-8') as f: return json.load(f) @pytest.fixture def validator_results(): - dir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(dir, + basedir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(basedir, '../', 'runtime/validation/faux-dev/result.json'), encoding='utf-8') as f: @@ -63,6 +67,5 @@ def test_dns_server_resolves(container_data): @pytest.mark.skip(reason='requires internet') def test_validator_results_compliant(validator_results): - results = [True if x['result'] == 'compliant' else False - for x in validator_results['results']] + results = [x['result'] == 'compliant' for x in validator_results['results']] assert all(results) diff --git a/testing/test_pylint b/testing/test_pylint index 5cd1dff73..3f4d8a3ed 100755 --- a/testing/test_pylint +++ b/testing/test_pylint @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ERROR_LIMIT=1100 +ERROR_LIMIT=175 sudo cmd/install @@ -38,4 +38,4 @@ if (( $new_errors > $ERROR_LIMIT)); then exit 1 fi -exit 0 +exit 0 \ No newline at end of file diff --git a/testing/test_tests b/testing/test_tests new file mode 100755 index 000000000..6ba9fef94 --- /dev/null +++ b/testing/test_tests @@ -0,0 +1,120 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o xtrace +ip a +TEST_DIR=/tmp/results +MATRIX=testing/test_tests.json + +mkdir -p $TEST_DIR + +# Setup requirements +sudo apt-get update +sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils isc-dhcp-client + +pip3 install pytest + +# Start OVS +# Setup device network +sudo ip link add dev endev0a type veth peer name endev0b +sudo ip link set dev endev0a up +sudo ip link set dev endev0b up +sudo docker network create -d macvlan -o parent=endev0b endev1 + +sudo /usr/share/openvswitch/scripts/ovs-ctl start + +# Build Test Container +sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f ./testing/docker/ci_test_device1/Dockerfile + +cat <local/system.json +{ + "network": { + "device_intf": "endev0a", + "internet_intf": "eth0" + }, + "log_level": "DEBUG", + "monitor_period": 30 +} +EOF + +sudo cmd/install + +TESTERS=$(jq -r 'keys[]' $MATRIX) +for tester in $TESTERS; do + testrun_log=$TEST_DIR/${tester}_testrun.log + device_log=$TEST_DIR/${tester}_device.log + + image=$(jq -r .$tester.image $MATRIX) + ethmac=$(jq -r .$tester.ethmac $MATRIX) + args=$(jq -r .$tester.args $MATRIX) + + touch $testrun_log + sudo timeout 900 cmd/start --single-intf > $testrun_log 2>&1 & + TPID=$! + + # Time to wait for testrun to be ready + WAITING=600 + for i in `seq 1 $WAITING`; do + tail -1 $testrun_log + if [[ -n $(fgrep "Waiting for devices on the network" $testrun_log) ]]; then + break + fi + + if [[ ! -d /proc/$TPID ]]; then + cat $testrun_log + echo "error encountered starting test run" + exit 1 + fi + + sleep 1 + done + + if [[ $i -eq $WAITING ]]; then + cat $testrun_log + echo "failed after waiting $WAITING seconds for test-run start" + exit 1 + fi + + # Load Test Container + sudo docker run -d \ + --network=endev1 \ + --mac-address=$ethmac \ + --cap-add=NET_ADMIN \ + -v /tmp:/out \ + --privileged \ + --name=$tester \ + ci_test_device1 $args + + wait $TPID + # Following line indicates that tests are completed but wait till it exits + # Completed running test modules on device with mac addr 7e:41:12:d2:35:6a + #Change this line! - LOGGER.info(f"""Completed running test modules on device + # with mac addr {device.mac_addr}""") + + ls runtime + more runtime/network/*.log + sudo docker kill $tester + sudo docker logs $tester | cat + + cp runtime/test/${ethmac//:/}/results.json $TEST_DIR/$tester.json + more $TEST_DIR/$tester.json + more $testrun_log + +done + +pytest -s testing/test_tests.py + +exit $? diff --git a/testing/test_tests.json b/testing/test_tests.json new file mode 100644 index 000000000..076e9149e --- /dev/null +++ b/testing/test_tests.json @@ -0,0 +1,19 @@ +{ + "tester1": { + "image": "test-run/ci_test1", + "args": "oddservices", + "ethmac": "02:42:aa:00:00:01", + "expected_results": { + "security.nmap.ports": "non-compliant" + } + }, + "tester2": { + "image": "test-run/ci_test1", + "args": "", + "ethmac": "02:42:aa:00:00:02", + "expected_results": { + "security.nmap.ports": "compliant" + } + } + +} \ No newline at end of file diff --git a/testing/test_tests.py b/testing/test_tests.py new file mode 100644 index 000000000..7c60484f0 --- /dev/null +++ b/testing/test_tests.py @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Test assertions for CI testing of tests """ +# Temporarily disabled because using Pytest fixtures +# TODO refactor fixtures to not trigger error +# pylint: disable=redefined-outer-name + +import json +import pytest +import os +import glob +import itertools + +from pathlib import Path +from dataclasses import dataclass + +TEST_MATRIX = 'test_tests.json' +RESULTS_PATH = '/tmp/results/*.json' + +@dataclass(frozen=True) +class TestResult: + name: str + result: str + __test__ = False + + +def collect_expected_results(expected_results): + """ Yields results from expected_results property of the test matrix""" + for name, result in expected_results.items(): + yield TestResult(name, result) + + +def collect_actual_results(results_dict): + """ Yields results from an already loaded testrun results file """ + # "module"."results".[list]."result" + for maybe_module, child in results_dict.items(): + if 'results' in child and maybe_module != 'baseline': + for test in child['results']: + yield TestResult(test['name'], test['result']) + + +@pytest.fixture +def test_matrix(): + basedir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(basedir, TEST_MATRIX), encoding='utf-8') as f: + return json.load(f) + + +@pytest.fixture +def results(): + results = {} + for file in [Path(x) for x in glob.glob(RESULTS_PATH)]: + with open(file, encoding='utf-8') as f: + results[file.stem] = json.load(f) + return results + + +def test_tests(results, test_matrix): + """ Check if each testers expect results were obtained """ + for tester, props in test_matrix.items(): + expected = set(collect_expected_results(props['expected_results'])) + actual = set(collect_actual_results(results[tester])) + + assert expected.issubset(actual), f'{tester} expected results not obtained' + +def test_list_tests(capsys, results, test_matrix): + all_tests = set(itertools.chain.from_iterable( + [collect_actual_results(results[x]) for x in results.keys()])) + + ci_pass = set([test + for testers in test_matrix.values() + for test, result in testers['expected_results'].items() + if result == 'compliant']) + + ci_fail = set([test + for testers in test_matrix.values() + for test, result in testers['expected_results'].items() + if result == 'non-compliant']) + + with capsys.disabled(): + print('============') + print('============') + print('tests seen:') + print('\n'.join([x.name for x in all_tests])) + print('\ntesting for pass:') + print('\n'.join(ci_pass)) + print('\ntesting for pass:') + print('\n'.join(ci_pass)) + + assert True diff --git a/testing/unit_test/run_tests.sh b/testing/unit_test/run_tests.sh new file mode 100644 index 000000000..5b1ed6257 --- /dev/null +++ b/testing/unit_test/run_tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e + +# This script should be run from within the unit_test directory. If +# it is run outside this directory, paths will not be resolved correctly. + +# Move into the root directory of test-run +pushd ../../ >/dev/null 2>&1 + +echo "Root Dir: $PWD" + +# Setup the python path +export PYTHONPATH="$PWD/framework/python/src" + +# Run the DHCP Unit tests +python3 -u $PWD/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py +python3 -u $PWD/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py + +popd >/dev/null 2>&1 \ No newline at end of file From 2724ed8706ba013cc08212bd2e63720c86fdaa4e Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 18 Jul 2023 09:56:46 +0100 Subject: [PATCH 10/32] Fix testing command --- testing/test_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_tests b/testing/test_tests index ed14f1043..574c9c89f 100755 --- a/testing/test_tests +++ b/testing/test_tests @@ -62,7 +62,7 @@ for tester in $TESTERS; do args=$(jq -r .$tester.args $MATRIX) touch $testrun_log - sudo timeout 900 cmd/start --single-intf > $testrun_log 2>&1 & + sudo timeout 900 bin/testrun --single-intf > $testrun_log 2>&1 & TPID=$! # Time to wait for testrun to be ready From 768ac101a9fe061bf8aacd513c7a209b24c1c2bf Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 18 Jul 2023 10:19:14 +0100 Subject: [PATCH 11/32] Disable API on testing --- testing/test_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_tests b/testing/test_tests index 574c9c89f..f4f4cda73 100755 --- a/testing/test_tests +++ b/testing/test_tests @@ -62,7 +62,7 @@ for tester in $TESTERS; do args=$(jq -r .$tester.args $MATRIX) touch $testrun_log - sudo timeout 900 bin/testrun --single-intf > $testrun_log 2>&1 & + sudo timeout 900 bin/testrun --single-intf --no-ui > $testrun_log 2>&1 & TPID=$! # Time to wait for testrun to be ready From 192bae867087427fc9714fbfcab0daf04eca0694 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 18 Jul 2023 15:19:30 +0100 Subject: [PATCH 12/32] Add API session --- bin/testrun | 4 +- framework/python/src/api/api.py | 75 +++++--- framework/python/src/api/system_config.py | 5 - .../python/src/api/system_network_config.py | 5 - framework/python/src/common/device.py | 6 +- framework/python/src/common/session.py | 153 +++++++++++++++- framework/python/src/core/testrun.py | 68 ++++--- framework/python/src/net_orc/listener.py | 8 +- .../python/src/net_orc/network_device.py | 24 --- .../src/net_orc/network_orchestrator.py | 167 +++++++----------- framework/python/src/net_orc/ovs_control.py | 33 +--- .../python/src/test_orc/test_orchestrator.py | 4 +- modules/test/base/python/src/test_module.py | 22 +-- resources/devices/template/device_config.json | 167 +----------------- 14 files changed, 332 insertions(+), 409 deletions(-) delete mode 100644 framework/python/src/api/system_config.py delete mode 100644 framework/python/src/api/system_network_config.py delete mode 100644 framework/python/src/net_orc/network_device.py diff --git a/bin/testrun b/bin/testrun index fde14451f..e991d4082 100755 --- a/bin/testrun +++ b/bin/testrun @@ -20,8 +20,8 @@ if [[ "$EUID" -ne 0 ]]; then fi # TODO: Obtain TESTRUNPATH from user environment variables -#TESTRUNPATH="/home/boddey/Desktop/test-run" -#cd $TESTRUNPATH +TESTRUNPATH="/home/boddey/Desktop/test-run" +cd $TESTRUNPATH # Ensure that /var/run/netns folder exists sudo mkdir -p /var/run/netns diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 40cda24be..1873393bd 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from fastapi import FastAPI, APIRouter, Response, status +from fastapi import FastAPI, APIRouter, Response, Request, status import json +from json import JSONDecodeError import psutil import threading import uvicorn -from api.system_config import SystemConfig from common import logger LOGGER = logger.get_logger("api") @@ -32,8 +32,7 @@ def __init__(self, test_run): self._name = "TestRun API" self._router = APIRouter() - self._devices = self._test_run.get_devices() - self._config_file_url = self._test_run.get_config_file() + self._session = self._test_run.get_session() self._router.add_api_route("/system/interfaces", self.get_sys_interfaces) self._router.add_api_route("/system/config", self.post_sys_config, @@ -72,48 +71,72 @@ async def get_sys_interfaces(self): ifaces.append(iface) return ifaces - async def post_sys_config(self, sys_config: SystemConfig): + async def post_sys_config(self, request: Request): + config = (await request.body()).decode("UTF-8") + config_json = json.loads(config) + self._session.set_config(config_json) + return self._session.get_config() - config_file = open(self._config_file_url, "r", encoding="utf-8") - json_contents = json.load(config_file) - config_file.close() + async def get_sys_config(self): + return self._session.get_config() - json_contents["network"]["device_intf"] = sys_config.network.device_intf - json_contents["network"]["internet_intf"] = sys_config.network.internet_intf + async def get_devices(self): + return self._session.get_device_repository() - with open(self._config_file_url, "w", encoding="utf-8") as config_file: - json.dump(json_contents, config_file, indent=2) + async def start_test_run(self, request: Request, response: Response): - return sys_config + LOGGER.debug("Received start command") - async def get_sys_config(self): - config_file = open(self._config_file_url, "r", encoding="utf-8") - json_contents = json.load(config_file) - config_file.close() - return json_contents + # Check request is valid + body = (await request.body()).decode("UTF-8") + body_json = None - async def get_devices(self): - return self._devices + try: + body_json = json.loads(body) + except JSONDecodeError: + response.status_code = status.HTTP_400_BAD_REQUEST + return json.loads('{"error": "Invalid JSON received"}') - async def start_test_run(self, response: Response): - LOGGER.debug("Received start command") - if self._test_run.get_session().status != "Idle": - LOGGER.debug("Test Run is already running. Cannot start another instance.") + if "device" not in body_json or "mac_addr" not in body_json["device"]: + response.status_code = status.HTTP_400_BAD_REQUEST + return json.loads('{"error": "Valid device MAC address has not been specified"}') + + device = self._session.get_device(body_json["device"]["mac_addr"]) + + # Check Test Run is not already running + if self._test_run.get_session().get_status() != "Idle": + LOGGER.debug("Test Run is already running. Cannot start another instance") response.status_code = status.HTTP_409_CONFLICT return json.loads('{"error": "Test Run is already running"}') + + # Check if requested device is known in the device repository + if device is None: + response.status_code = status.HTTP_404_NOT_FOUND + return json.loads('{"error": "A device with that MAC address could not be found"}') + + # Check Test Run is able to start + if self._test_run.get_net_orc().check_config() is False: + response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + return json.loads('{"error": "Configured interfaces are not ready for use. Ensure both interfaces are connected."}') + + self._test_run.get_session().set_target_device(device) + LOGGER.info(f"Starting Test Run with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}") + thread = threading.Thread(target=self._start_test_run, name="Test Run") thread.start() - return json.loads('{"status": "Starting Test Run"}') + return self._test_run.get_session().to_json() def _start_test_run(self): self._test_run.start() async def stop_test_run(self): LOGGER.info("Received stop command. Stopping Test Run") + self._test_run.stop() + return json.loads('{"success": "Test Run stopped"}') async def get_status(self): - return self._test_run.get_session() + return self._test_run.get_session().to_json() async def get_history(self): LOGGER.info("Returning previous Test Runs to UI") diff --git a/framework/python/src/api/system_config.py b/framework/python/src/api/system_config.py deleted file mode 100644 index 75ee2fb9d..000000000 --- a/framework/python/src/api/system_config.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel -from api.system_network_config import SystemNetworkConfig - -class SystemConfig(BaseModel): - network: SystemNetworkConfig diff --git a/framework/python/src/api/system_network_config.py b/framework/python/src/api/system_network_config.py deleted file mode 100644 index 78a511491..000000000 --- a/framework/python/src/api/system_network_config.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - -class SystemNetworkConfig(BaseModel): - device_intf: str - internet_intf: str diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index efce2dba1..b70099519 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -14,14 +14,14 @@ """Track device object information.""" -from net_orc.network_device import NetworkDevice from dataclasses import dataclass - @dataclass -class Device(NetworkDevice): +class Device(): """Represents a physical device and it's configuration.""" + mac_addr: str = None manufacturer: str = None model: str = None test_modules: str = None + ip_addr: str = None diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 155c45361..91abd5a26 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -14,15 +14,152 @@ """Track testing status.""" -from dataclasses import dataclass, field -from common.device import Device +import datetime +import json +import os + +NETWORK_KEY = 'network' +DEVICE_INTF_KEY = 'device_intf' +INTERNET_INTF_KEY = 'internet_intf' +RUNTIME_KEY = 'runtime' +MONITOR_PERIOD_KEY = 'monitor_period' +STARTUP_TIMEOUT_KEY = 'startup_timeout' +LOG_LEVEL_KEY = 'log_level' -@dataclass class TestRunSession(): """Represents the current session of Test Run.""" - status: str = 'Idle' - device: Device = None - started: str = None - finished: str = None - tests: list = field(default_factory=list) + def __init__(self, config_file): + self._status = 'Idle' + self._device = None + self._started = None + self._finished = None + self._tests = [] + + self._config_file = config_file + + self._config = self._get_default_config() + self._load_config() + + self._device_repository = [] + + def start(self): + self._status = 'Starting' + self._started = datetime.datetime.now() + + def get_started(self): + return self._started + + def get_finished(self): + return self._finished + + def _get_default_config(self): + return { + 'network': { + 'device_intf': '', + 'internet_intf': '' + }, + 'log_level': 'INFO', + 'startup_timeout': 60, + 'monitor_period': 30, + 'runtime': 120 + } + + def get_config(self): + return self._config + + def _load_config(self): + + if not os.path.isfile(self._config_file): + return + + with open(self._config_file, 'r', encoding='utf-8') as f: + config_file_json = json.load(f) + + # Network interfaces + if NETWORK_KEY in config_file_json: + if DEVICE_INTF_KEY in config_file_json[NETWORK_KEY]: + self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json[NETWORK_KEY][DEVICE_INTF_KEY] + if INTERNET_INTF_KEY in config_file_json[NETWORK_KEY]: + self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json[NETWORK_KEY][INTERNET_INTF_KEY] + + if RUNTIME_KEY in config_file_json: + self._config[RUNTIME_KEY] = config_file_json[RUNTIME_KEY] + + if STARTUP_TIMEOUT_KEY in config_file_json: + self._config[STARTUP_TIMEOUT_KEY] = config_file_json[STARTUP_TIMEOUT_KEY] + + if MONITOR_PERIOD_KEY in config_file_json: + self._config[MONITOR_PERIOD_KEY] = config_file_json[MONITOR_PERIOD_KEY] + + if LOG_LEVEL_KEY in config_file_json: + self._config[LOG_LEVEL_KEY] = config_file_json[LOG_LEVEL_KEY] + + def _save_config(self): + with open(self._config_file, 'w', encoding='utf-8') as f: + f.write(json.dumps(self._config, indent=2)) + + def get_runtime(self): + return self._config[RUNTIME_KEY] + + def get_log_level(self): + return self._config[LOG_LEVEL_KEY] + + def get_device_interface(self): + return self._config[NETWORK_KEY][DEVICE_INTF_KEY] + + def get_internet_interface(self): + return self._config[NETWORK_KEY][INTERNET_INTF_KEY] + + def get_monitor_period(self): + return self._config[MONITOR_PERIOD_KEY] + + def get_startup_timeout(self): + return self._config[STARTUP_TIMEOUT_KEY] + + def set_config(self, config_json): + self._config = config_json + self._save_config() + + def set_target_device(self, device): + self._device = device + + def get_target_device(self): + return self._device + + def get_device_repository(self): + return self._device_repository + + def add_device(self, device): + self._device_repository.append(device) + + def get_device(self, mac_addr): + for device in self._device_repository: + if device.mac_addr == mac_addr: + return device + return None + + def get_status(self): + return self._status + + def set_status(self, status): + self._status = status + + def get_tests(self): + return self._tests + + def reset(self): + self.set_status('Idle') + self.set_target_device(None) + self._tests = [] + self._started = None + self._finished = None + + def to_json(self): + return { + 'status': self.get_status(), + 'device': self.get_target_device(), + 'started': self.get_started(), + 'finished': self.get_finished(), + 'tests': self.get_tests() + } diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 4f424a185..a66c2c859 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -42,11 +42,13 @@ from net_orc import network_orchestrator as net_orc # pylint: disable=wrong-import-position LOGGER = logger.get_logger('test_run') + DEFAULT_CONFIG_FILE = 'local/system.json' EXAMPLE_CONFIG_FILE = 'local/system.json.example' -RUNTIME = 120 + LOCAL_DEVICES_DIR = 'local/devices' RESOURCE_DEVICES_DIR = 'resources/devices' + DEVICE_CONFIG = 'device_config.json' DEVICE_MANUFACTURER = 'manufacturer' DEVICE_MODEL = 'model' @@ -68,30 +70,28 @@ def __init__(self, no_ui=False): if config_file is None: - config_file = DEFAULT_CONFIG_FILE + self._config_file = self._get_config_abs(DEFAULT_CONFIG_FILE) + else: + self._config_file = self._get_config_abs(config_file) self._devices = [] self._net_only = net_only self._single_intf = single_intf - self._config_file = config_file self._no_ui = no_ui - self._session = TestRunSession() - # Catch any exit signals self._register_exits() - # Expand the config file to absolute pathing - config_file_abs = self._get_config_abs(config_file=self._config_file) - + self._session = TestRunSession(config_file=self._config_file) self._load_all_devices() self._net_orc = net_orc.NetworkOrchestrator( - config_file=config_file_abs, + session=self._session, validate=validate, single_intf = self._single_intf) - - self._test_orc = test_orc.TestOrchestrator(self._net_orc) + self._test_orc = test_orc.TestOrchestrator( + self._session, + self._net_orc) if self._no_ui: self.start() @@ -128,63 +128,69 @@ def _load_devices(self, device_dir): model=device_model, mac_addr=mac_addr, test_modules=test_modules) - self._devices.append(device) + self.get_session().add_device(device) def start(self): - self._set_status('Starting') + self._session.start() self._start_network() if self._net_only: LOGGER.info('Network only option configured, no tests will be run') - self._net_orc.listener.register_callback( + self.get_net_orc().listener.register_callback( self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED] ) - self._net_orc.start_listener() + self.get_net_orc().start_listener() LOGGER.info('Waiting for devices on the network...') while True: - time.sleep(RUNTIME) + time.sleep(self._session.get_runtime()) else: self._test_orc.start() - self._net_orc.listener.register_callback( + self.get_net_orc().get_listener().register_callback( self._device_stable, [NetworkEvent.DEVICE_STABLE] ) - self._net_orc.listener.register_callback( + self.get_net_orc().get_listener().register_callback( self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED] ) - self._net_orc.start_listener() + self.get_net_orc().start_listener() self._set_status('Waiting for device') LOGGER.info('Waiting for devices on the network...') - time.sleep(RUNTIME) + time.sleep(self._session.get_runtime()) if not (self._test_orc.test_in_progress() or - self._net_orc.monitor_in_progress()): + self.get_net_orc().monitor_in_progress()): LOGGER.info('''Timed out whilst waiting for device or stopping due to test completion''') else: while (self._test_orc.test_in_progress() or - self._net_orc.monitor_in_progress()): + self.get_net_orc().monitor_in_progress()): time.sleep(5) self.stop() def stop(self, kill=False): self._set_status('Stopping') + + # Prevent discovering new devices whilst stopping + if self.get_net_orc().get_listener() is not None: + self.get_net_orc().get_listener().stop_listener() + self._stop_tests() self._stop_network(kill=kill) - self._set_status('Idle') + + self.get_session().reset() def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -210,17 +216,17 @@ def _get_config_abs(self, config_file=None): def get_config_file(self): return self._get_config_abs() - def get_devices(self): - return self._devices + def get_net_orc(self): + return self._net_orc def _start_network(self): # Start the network orchestrator - if not self._net_orc.start(): + if not self.get_net_orc().start(): self.stop(kill=True) sys.exit(1) def _stop_network(self, kill=False): - self._net_orc.stop(kill=kill) + self.get_net_orc().stop(kill=kill) def _stop_tests(self): self._test_orc.stop() @@ -232,6 +238,12 @@ def get_device(self, mac_addr): return device def _device_discovered(self, mac_addr): + + if self.get_session().get_target_device() is not None: + if mac_addr != self.get_session().get_target_device().mac_addr: + # Ignore discovered device + return + self._set_status('Identifying device') device = self.get_device(mac_addr) if device is not None: @@ -251,7 +263,7 @@ def _device_stable(self, mac_addr): self._set_status('Complete') def _set_status(self, status): - self._session.status = status + self._session.set_status(status) def get_session(self): return self._session diff --git a/framework/python/src/net_orc/listener.py b/framework/python/src/net_orc/listener.py index 4f8e1961f..83805f908 100644 --- a/framework/python/src/net_orc/listener.py +++ b/framework/python/src/net_orc/listener.py @@ -31,8 +31,9 @@ class Listener: """Methods to start and stop the network listener.""" - def __init__(self, device_intf): - self._device_intf = device_intf + def __init__(self, session): + self._session = session + self._device_intf = self._session.get_device_interface() self._device_intf_mac = get_if_hwaddr(self._device_intf) self._sniffer = AsyncSniffer(iface=self._device_intf, @@ -47,7 +48,8 @@ def start_listener(self): def stop_listener(self): """Stop sniffing packets on the device interface.""" - self._sniffer.stop() + if self._sniffer.running: + self._sniffer.stop() def is_running(self): """Determine whether the sniffer is running.""" diff --git a/framework/python/src/net_orc/network_device.py b/framework/python/src/net_orc/network_device.py deleted file mode 100644 index f17ac0f0d..000000000 --- a/framework/python/src/net_orc/network_device.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Track device object information.""" -from dataclasses import dataclass - - -@dataclass -class NetworkDevice: - """Represents a physical device and it's configuration.""" - - mac_addr: str - ip_addr: str = None diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 885d5f75a..7651b9914 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -14,7 +14,6 @@ """Network orchestrator is responsible for managing all of the virtual network services""" -import getpass import ipaddress import json import os @@ -26,54 +25,40 @@ from docker.types import Mount from common import logger, util from net_orc.listener import Listener -from net_orc.network_device import NetworkDevice from net_orc.network_event import NetworkEvent from net_orc.network_validator import NetworkValidator from net_orc.ovs_control import OVSControl from net_orc.ip_control import IPControl LOGGER = logger.get_logger('net_orc') -CONFIG_FILE = 'local/system.json' -EXAMPLE_CONFIG_FILE = 'local/system.json.example' RUNTIME_DIR = 'runtime' TEST_DIR = 'test' -MONITOR_PCAP = 'monitor.pcap' NET_DIR = 'runtime/network' NETWORK_MODULES_DIR = 'modules/network' + +MONITOR_PCAP = 'monitor.pcap' NETWORK_MODULE_METADATA = 'conf/module_config.json' + DEVICE_BRIDGE = 'tr-d' INTERNET_BRIDGE = 'tr-c' PRIVATE_DOCKER_NET = 'tr-private-net' CONTAINER_NAME = 'network_orchestrator' -RUNTIME_KEY = 'runtime' -MONITOR_PERIOD_KEY = 'monitor_period' -STARTUP_TIMEOUT_KEY = 'startup_timeout' -DEFAULT_STARTUP_TIMEOUT = 60 -DEFAULT_RUNTIME = 1200 -DEFAULT_MONITOR_PERIOD = 300 - class NetworkOrchestrator: """Manage and controls a virtual testing network.""" def __init__(self, - config_file=CONFIG_FILE, + session, validate=True, single_intf=False): - self._runtime = DEFAULT_RUNTIME - self._startup_timeout = DEFAULT_STARTUP_TIMEOUT - self._monitor_period = DEFAULT_MONITOR_PERIOD + self._session = session self._monitor_in_progress = False - - self._int_intf = None - self._dev_intf = None + self._validate = validate self._single_intf = single_intf - self.listener = None + self._listener = None self._net_modules = [] - self._devices = [] - self.validate = validate self._path = os.path.dirname( os.path.dirname( @@ -83,8 +68,7 @@ def __init__(self, self.validator = NetworkValidator() shutil.rmtree(os.path.join(os.getcwd(), NET_DIR), ignore_errors=True) self.network_config = NetworkConfig() - self.load_config(config_file) - self._ovs = OVSControl() + self._ovs = OVSControl(self._session) self._ip_ctrl = IPControl() def start(self): @@ -100,28 +84,42 @@ def start(self): # Restore the network first if required self.stop(kill=True) - return self.start_network() + self.start_network() + + return True + + def check_config(self): + + self._import_config() + + if not util.interface_exists(self._int_intf) or not util.interface_exists( + self._dev_intf): + LOGGER.error('Configured interfaces are not ready for use. ' + + 'Ensure both interfaces are connected.') + return False + return True def start_network(self): """Start the virtual testing network.""" LOGGER.info('Starting network') self.build_network_modules() - if not self.create_net(): - return False + + self.create_net() self.start_network_services() - if self.validate: + if self._validate: # Start the validator after network is ready self.validator.start() # Get network ready (via Network orchestrator) LOGGER.debug('Network is ready') - return True + def get_listener(self): + return self._listener def start_listener(self): - self.listener.start_listener() + self.get_listener().start_listener() def stop(self, kill=False): """Stop the network orchestrator.""" @@ -139,43 +137,35 @@ def stop_network(self, kill=False): self.stop_networking_services(kill=kill) self.restore_net() - def load_config(self, config_file=None): - if config_file is None: - # If not defined, use relative pathing to local file - self._config_file = os.path.join(self._path, CONFIG_FILE) - else: - # If defined, use as provided - self._config_file = config_file - - if not os.path.isfile(self._config_file): - LOGGER.error('Configuration file is not present at ' + config_file) - LOGGER.info('An example is present in ' + EXAMPLE_CONFIG_FILE) - sys.exit(1) + def _device_discovered(self, mac_addr): - LOGGER.info('Loading config file: ' + os.path.abspath(self._config_file)) - with open(self._config_file, encoding='UTF-8') as config_json_file: - config_json = json.load(config_json_file) - self.import_config(config_json) + device = self._session.get_device(mac_addr) - def _device_discovered(self, mac_addr): + if self._session.get_target_device() is not None: + if mac_addr != self._session.get_target_device().mac_addr: + # Ignore discovered device + return self._monitor_in_progress = True LOGGER.debug( f'Discovered device {mac_addr}. Waiting for device to obtain IP') - device = self._get_device(mac_addr=mac_addr) + if device is None: + LOGGER.debug(f'Device with MAC address {mac_addr} does not exist in device repository') + # Ignore device if not registered + return device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, - device.mac_addr.replace(':', '')) + mac_addr.replace(':', '')) os.makedirs(device_runtime_dir) util.run_command(f'chown -R {self._host_user} {device_runtime_dir}') - packet_capture = sniff(iface=self._dev_intf, - timeout=self._startup_timeout, + packet_capture = sniff(iface=self._session.get_device_interface(), + timeout=self._session.get_startup_timeout(), stop_filter=self._device_has_ip) wrpcap( - os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), + os.path.join(RUNTIME_DIR, TEST_DIR, mac_addr.replace(':', ''), 'startup.pcap'), packet_capture) if device.ip_addr is None: @@ -192,49 +182,35 @@ def monitor_in_progress(self): return self._monitor_in_progress def _device_has_ip(self, packet): - device = self._get_device(mac_addr=packet.src) + device = self._session.get_device(mac_addr=packet.src) if device is None or device.ip_addr is None: return False return True def _dhcp_lease_ack(self, packet): mac_addr = packet[BOOTP].chaddr.hex(':')[0:17] - device = self._get_device(mac_addr=mac_addr) + device = self._session.get_device(mac_addr=mac_addr) + + # Ignore devices that are not registered + if device is None: + return + + # TODO: Check if device is None device.ip_addr = packet[BOOTP].yiaddr def _start_device_monitor(self, device): """Start a timer until the steady state has been reached and callback the steady state method for this device.""" LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' - f'for {str(self._monitor_period)} seconds') + f'for {str(self._session.get_monitor_period())} seconds') - packet_capture = sniff(iface=self._dev_intf, timeout=self._monitor_period) + packet_capture = sniff(iface=self._session.get_device_interface(), timeout=self._session.get_monitor_period()) wrpcap( os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), 'monitor.pcap'), packet_capture) self._monitor_in_progress = False - self.listener.call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) - - def _get_device(self, mac_addr): - for device in self._devices: - if device.mac_addr == mac_addr: - return device - - device = NetworkDevice(mac_addr=mac_addr) - self._devices.append(device) - return device - - def import_config(self, json_config): - self._int_intf = json_config['network']['internet_intf'] - self._dev_intf = json_config['network']['device_intf'] - - if RUNTIME_KEY in json_config: - self._runtime = json_config[RUNTIME_KEY] - if STARTUP_TIMEOUT_KEY in json_config: - self._startup_timeout = json_config[STARTUP_TIMEOUT_KEY] - if MONITOR_PERIOD_KEY in json_config: - self._monitor_period = json_config[MONITOR_PERIOD_KEY] + self.get_listener().call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) def _check_network_services(self): LOGGER.debug('Checking network modules...') @@ -319,17 +295,11 @@ def _ci_post_network_create(self): def create_net(self): LOGGER.info('Creating baseline network') - if not util.interface_exists(self._int_intf) or not util.interface_exists( - self._dev_intf): - LOGGER.error('Configured interfaces are not ready for use. ' + - 'Ensure both interfaces are connected.') - return False - if self._single_intf: self._ci_pre_network_create() # Remove IP from internet adapter - util.run_command('ifconfig ' + self._int_intf + ' 0.0.0.0') + util.run_command('ifconfig ' + self._session.get_internet_interface() + ' 0.0.0.0') # Setup the virtual network if not self._ovs.create_baseline_net(verify=True): @@ -342,14 +312,12 @@ def create_net(self): self._create_private_net() - self.listener = Listener(self._dev_intf) - self.listener.register_callback(self._device_discovered, + self._listener = Listener(self._session) + self.get_listener().register_callback(self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED]) - self.listener.register_callback(self._dhcp_lease_ack, + self.get_listener().register_callback(self._dhcp_lease_ack, [NetworkEvent.DHCP_LEASE_ACK]) - return True - def load_network_modules(self): """Load network modules from module_config.json.""" LOGGER.debug('Loading network modules from /' + NETWORK_MODULES_DIR) @@ -666,9 +634,8 @@ def restore_net(self): LOGGER.info('Clearing baseline network') - if hasattr(self, 'listener' - ) and self.listener is not None and self.listener.is_running(): - self.listener.stop_listener() + if hasattr(self, 'listener') and self.get_listener() is not None and self.get_listener().is_running(): + self.get_listener().stop_listener() client = docker.from_env() @@ -686,10 +653,12 @@ def restore_net(self): # Clean up any existing network artifacts self._ip_ctrl.clean_all() + internet_intf = self._session.get_internet_interface() + # Restart internet interface - if util.interface_exists(self._int_intf): - util.run_command('ip link set ' + self._int_intf + ' down') - util.run_command('ip link set ' + self._int_intf + ' up') + if util.interface_exists(internet_intf): + util.run_command('ip link set ' + internet_intf + ' down') + util.run_command('ip link set ' + internet_intf + ' up') LOGGER.info('Network is restored') @@ -717,10 +686,6 @@ def __init__(self): self.net_config = NetworkModuleNetConfig() - -# The networking configuration for a network module - - class NetworkModuleNetConfig: """Define all the properties of the network config for a network module""" @@ -743,10 +708,6 @@ def get_ipv4_addr_with_prefix(self): def get_ipv6_addr_with_prefix(self): return format(self.ipv6_address) + '/' + str(self.ipv6_network.prefixlen) - -# Represents the current configuration of the network for the device bridge - - class NetworkConfig: """Define all the properties of the network configuration""" diff --git a/framework/python/src/net_orc/ovs_control.py b/framework/python/src/net_orc/ovs_control.py index 83823e8fa..c48e58e3b 100644 --- a/framework/python/src/net_orc/ovs_control.py +++ b/framework/python/src/net_orc/ovs_control.py @@ -18,7 +18,6 @@ from common import logger from common import util -CONFIG_FILE = 'local/system.json' DEVICE_BRIDGE = 'tr-d' INTERNET_BRIDGE = 'tr-c' LOGGER = logger.get_logger('ovs_ctrl') @@ -27,10 +26,8 @@ class OVSControl: """OVS Control""" - def __init__(self): - self._int_intf = None - self._dev_intf = None - self._load_config() + def __init__(self, session): + self._session = session def add_bridge(self, bridge_name): LOGGER.debug('Adding OVS bridge: ' + bridge_name) @@ -80,11 +77,11 @@ def validate_baseline_network(self): LOGGER.debug('Validating baseline network') # Verify the device bridge - dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._dev_intf]) + dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._session.get_device_interface()]) LOGGER.debug('Device bridge verified: ' + str(dev_bridge)) # Verify the internet bridge - int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._int_intf]) + int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()]) LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) return dev_bridge and int_bridge @@ -107,7 +104,7 @@ def create_baseline_net(self, verify=True): LOGGER.debug('Creating baseline network') # Remove IP from internet adapter - self.set_interface_ip(interface=self._int_intf, ip_addr='0.0.0.0') + self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0') # Create data plane self.add_bridge(DEVICE_BRIDGE) @@ -116,11 +113,11 @@ def create_baseline_net(self, verify=True): self.add_bridge(INTERNET_BRIDGE) # Remove IP from internet adapter - self.set_interface_ip(self._int_intf, '0.0.0.0') + self.set_interface_ip(self._session.get_internet_interface(), '0.0.0.0') # Add external interfaces to data and control plane - self.add_port(self._dev_intf, DEVICE_BRIDGE) - self.add_port(self._int_intf, INTERNET_BRIDGE) + self.add_port(self._session.get_device_interface(), DEVICE_BRIDGE) + self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE) # Enable forwarding of eapol packets self.add_flow(bridge_name=DEVICE_BRIDGE, @@ -145,20 +142,6 @@ def delete_bridge(self, bridge_name): success = util.run_command('ovs-vsctl --if-exists del-br ' + bridge_name) return success - def _load_config(self): - path = os.path.dirname(os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) - config_file = os.path.join(path, CONFIG_FILE) - LOGGER.debug('Loading configuration: ' + config_file) - with open(config_file, 'r', encoding='utf-8') as conf_file: - config_json = json.load(conf_file) - self._int_intf = config_json['network']['internet_intf'] - self._dev_intf = config_json['network']['device_intf'] - LOGGER.debug('Configuration loaded') - LOGGER.debug('Internet interface: ' + self._int_intf) - LOGGER.debug('Device interface: ' + self._dev_intf) - def restore_net(self): LOGGER.debug('Restoring network...') # Delete data plane diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index fef4e5bb5..03bcfd0bb 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -32,9 +32,10 @@ class TestOrchestrator: """Manages and controls the test modules.""" - def __init__(self, net_orc): + def __init__(self, session, net_orc): self._test_modules = [] self._module_config = None + self._session = session self._net_orc = net_orc self._test_in_progress = False @@ -45,7 +46,6 @@ def __init__(self, net_orc): # Resolve the path to the test-run folder #self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) - self._root_path = os.path.dirname(os.path.dirname( os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index e949976fa..6ff4f815b 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -61,8 +61,6 @@ def _get_device_tests(self, device_test_module): if 'tests' in device_test_module: if test['name'] in device_test_module['tests']: dev_test_config = device_test_module['tests'][test['name']] - if 'enabled' in dev_test_config: - test['enabled'] = dev_test_config['enabled'] if 'config' in test and 'config' in dev_test_config: test['config'].update(dev_test_config['config']) return module_tests @@ -83,19 +81,17 @@ def run_tests(self): test_method_name = '_' + test['name'].replace('.', '_') result = None test['start'] = datetime.now().isoformat() - if ('enabled' in test and test['enabled']) or 'enabled' not in test: - LOGGER.info('Attempting to run test: ' + test['name']) - # Resolve the correct python method by test name and run test - if hasattr(self, test_method_name): - if 'config' in test: - result = getattr(self, test_method_name)(config=test['config']) - else: - result = getattr(self, test_method_name)() + LOGGER.info('Attempting to run test: ' + test['name']) + # Resolve the correct python method by test name and run test + if hasattr(self, test_method_name): + if 'config' in test: + result = getattr(self, test_method_name)(config=test['config']) else: - LOGGER.info(f'Test {test["name"]} not resolved. Skipping') - result = None + result = getattr(self, test_method_name)() else: - LOGGER.info(f'Test {test["name"]} disabled. Skipping') + LOGGER.info(f'Test {test["name"]} not resolved. Skipping') + result = None + if result is not None: if isinstance(result, bool): test['result'] = 'compliant' if result else 'non-compliant' diff --git a/resources/devices/template/device_config.json b/resources/devices/template/device_config.json index 1e92de25d..7ee63cf95 100644 --- a/resources/devices/template/device_config.json +++ b/resources/devices/template/device_config.json @@ -4,176 +4,19 @@ "mac_addr": "aa:bb:cc:dd:ee:ff", "test_modules": { "dns": { - "enabled": true, - "tests": { - "dns.network.from_device": { - "enabled": true - }, - "dns.network.from_dhcp": { - "enabled": true - } - } + "enabled": true }, "connection": { - "enabled": true, - "tests": { - "connection.mac_address": { - "enabled": true - }, - "connection.mac_oui": { - "enabled": true - }, - "connection.target_ping": { - "enabled": true - } - , - "connection.single_ip": { - "enabled": true - } - } + "enabled": true }, "ntp": { - "enabled": true, - "tests": { - "ntp.network.ntp_support": { - "enabled": true - }, - "ntp.network.ntp_dhcp": { - "enabled": true - } - } + "enabled": true }, "baseline": { - "enabled": false, - "tests": { - "baseline.non-compliant": { - "enabled": true - }, - "baseline.pass": { - "enabled": true - }, - "baseline.skip": { - "enabled": true - } - } + "enabled": false }, "nmap": { - "enabled": true, - "tests": { - "security.nmap.ports": { - "enabled": true, - "security.services.ftp": { - "tcp_ports": { - "20": { - "allowed": false - }, - "21": { - "allowed": false - } - } - }, - "security.services.ssh": { - "tcp_ports": { - "22": { - "allowed": true - } - } - }, - "security.services.telnet": { - "tcp_ports": { - "23": { - "allowed": false - } - } - }, - "security.services.smtp": { - "tcp_ports": { - "25": { - "allowed": false - }, - "465": { - "allowed": false - }, - "587": { - "allowed": false - } - } - }, - "security.services.http": { - "tcp_ports": { - "80": { - "allowed": false - }, - "443": { - "allowed": true - } - } - }, - "security.services.pop": { - "tcp_ports": { - "110": { - "allowed": false - } - } - }, - "security.services.imap": { - "tcp_ports": { - "143": { - "allowed": false - } - } - }, - "security.services.snmpv3": { - "tcp_ports": { - "161": { - "allowed": false - }, - "162": { - "allowed": false - } - }, - "udp_ports": { - "161": { - "allowed": false - }, - "162": { - "allowed": false - } - } - }, - "security.services.https": { - "tcp_ports": { - "80": { - "allowed": false - } - } - }, - "security.services.vnc": { - "tcp_ports": { - "5500": { - "allowed": false - }, - "5800": { - "allowed": false - } - } - }, - "security.services.tftp": { - "udp_ports": { - "69": { - "allowed": false - } - } - }, - "security.services.ntp": { - "udp_ports": { - "123": { - "allowed": false - } - } - } - } - } + "enabled": true } } } From c176bcf5c504191f80aea348e997c9a502d9e703 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 18 Jul 2023 15:21:19 +0100 Subject: [PATCH 13/32] Remove old method --- framework/python/src/net_orc/network_orchestrator.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 7651b9914..becd3b746 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -90,10 +90,8 @@ def start(self): def check_config(self): - self._import_config() - - if not util.interface_exists(self._int_intf) or not util.interface_exists( - self._dev_intf): + if not util.interface_exists(self._session.get_internet_interface()) or not util.interface_exists( + self._session.get_device_interface()): LOGGER.error('Configured interfaces are not ready for use. ' + 'Ensure both interfaces are connected.') return False From 664706e1cd128bcfa682aef6a62609551ae62f0a Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 18 Jul 2023 15:22:05 +0100 Subject: [PATCH 14/32] Remove local vars --- bin/testrun | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/testrun b/bin/testrun index e991d4082..13a12010c 100755 --- a/bin/testrun +++ b/bin/testrun @@ -20,8 +20,8 @@ if [[ "$EUID" -ne 0 ]]; then fi # TODO: Obtain TESTRUNPATH from user environment variables -TESTRUNPATH="/home/boddey/Desktop/test-run" -cd $TESTRUNPATH +# TESTRUNPATH="/home/boddey/Desktop/test-run" +# cd $TESTRUNPATH # Ensure that /var/run/netns folder exists sudo mkdir -p /var/run/netns From 73d5a18875d02cb54cb158c238b0622a17c6322d Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 18 Jul 2023 15:36:47 +0100 Subject: [PATCH 15/32] Replace old var --- .../python/src/net_orc/network_orchestrator.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index becd3b746..7d550d4ae 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -255,30 +255,30 @@ def _ci_pre_network_create(self): """ self._ethmac = subprocess.check_output( - f'cat /sys/class/net/{self._int_intf}/address', + f'cat /sys/class/net/{self._session.get_internet_interface()}/address', shell=True).decode('utf-8').strip() self._gateway = subprocess.check_output( 'ip route | head -n 1 | awk \'{print $3}\'', shell=True).decode('utf-8').strip() self._ipv4 = subprocess.check_output( - f'ip a show {self._int_intf} | grep \"inet \" | awk \'{{print $2}}\'', + f'ip a show {self._session.get_internet_interface()} | grep \"inet \" | awk \'{{print $2}}\'', shell=True).decode('utf-8').strip() self._ipv6 = subprocess.check_output( - f'ip a show {self._int_intf} | grep inet6 | awk \'{{print $2}}\'', + f'ip a show {self._session.get_internet_interface()} | grep inet6 | awk \'{{print $2}}\'', shell=True).decode('utf-8').strip() self._brd = subprocess.check_output( - f'ip a show {self._int_intf} | grep \"inet \" | awk \'{{print $4}}\'', + f'ip a show {self._session.get_internet_interface()} | grep \"inet \" | awk \'{{print $4}}\'', shell=True).decode('utf-8').strip() def _ci_post_network_create(self): """ Restore network connection in CI environment """ LOGGER.info('post cr') - util.run_command(f'ip address del {self._ipv4} dev {self._int_intf}') - util.run_command(f'ip -6 address del {self._ipv6} dev {self._int_intf}') + util.run_command(f'ip address del {self._ipv4} dev {self._session.get_internet_interface()}') + util.run_command(f'ip -6 address del {self._ipv6} dev {self._session.get_internet_interface()}') util.run_command( - f'ip link set dev {self._int_intf} address 00:B0:D0:63:C2:26') - util.run_command(f'ip addr flush dev {self._int_intf}') - util.run_command(f'ip addr add dev {self._int_intf} 0.0.0.0') + f'ip link set dev {self._session.get_internet_interface()} address 00:B0:D0:63:C2:26') + util.run_command(f'ip addr flush dev {self._session.get_internet_interface()}') + util.run_command(f'ip addr add dev {self._session.get_internet_interface()} 0.0.0.0') util.run_command( f'ip addr add dev {INTERNET_BRIDGE} {self._ipv4} broadcast {self._brd}') util.run_command(f'ip -6 addr add {self._ipv6} dev {INTERNET_BRIDGE} ') From f0e13500fd5491f82dde43e4a1333fcc57f83e37 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 09:53:48 +0100 Subject: [PATCH 16/32] Add device config --- testing/test_tests | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/testing/test_tests b/testing/test_tests index f4f4cda73..c48e21e88 100755 --- a/testing/test_tests +++ b/testing/test_tests @@ -61,6 +61,14 @@ for tester in $TESTERS; do ethmac=$(jq -r .$tester.ethmac $MATRIX) args=$(jq -r .$tester.args $MATRIX) + cat <local/devices/$tester/device_config.json + { + "manufacturer": "Google", + "model": "$args", + "mac_addr": "$ethmac" + } + EOF + touch $testrun_log sudo timeout 900 bin/testrun --single-intf --no-ui > $testrun_log 2>&1 & TPID=$! From 6d9b5e57fbfbd6cf26811fc99d82f580f66d00e3 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 10:22:42 +0100 Subject: [PATCH 17/32] Add device configs --- .github/workflows/testing.yml | 6 ++--- testing/{ => baseline}/test_baseline | 0 testing/{ => baseline}/test_baseline.py | 0 .../device_configs/tester1/device_config.json | 22 +++++++++++++++++++ .../device_configs/tester2/device_config.json | 22 +++++++++++++++++++ testing/{ => pylint}/test_pylint | 0 testing/{ => tests}/example/mac | 0 testing/{ => tests}/example/mac1/results.json | 0 testing/{ => tests}/test_tests | 10 ++------- testing/{ => tests}/test_tests.json | 0 testing/{ => tests}/test_tests.py | 0 testing/{unit_test => unit}/run_tests.sh | 0 12 files changed, 49 insertions(+), 11 deletions(-) rename testing/{ => baseline}/test_baseline (100%) rename testing/{ => baseline}/test_baseline.py (100%) create mode 100644 testing/device_configs/tester1/device_config.json create mode 100644 testing/device_configs/tester2/device_config.json rename testing/{ => pylint}/test_pylint (100%) rename testing/{ => tests}/example/mac (100%) rename testing/{ => tests}/example/mac1/results.json (100%) rename testing/{ => tests}/test_tests (95%) rename testing/{ => tests}/test_tests.json (100%) rename testing/{ => tests}/test_tests.py (100%) rename testing/{unit_test => unit}/run_tests.sh (100%) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c981dbd56..9e6f35323 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v2.3.4 - name: Run tests shell: bash {0} - run: testing/test_baseline + run: testing/baseline/test_baseline testrun_tests: name: Tests @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v2.3.4 - name: Run tests shell: bash {0} - run: testing/test_tests + run: testing/tests/test_tests pylint: name: Pylint runs-on: ubuntu-22.04 @@ -37,4 +37,4 @@ jobs: uses: actions/checkout@v2.3.4 - name: Run tests shell: bash {0} - run: testing/test_pylint + run: testing/pylint/test_pylint diff --git a/testing/test_baseline b/testing/baseline/test_baseline similarity index 100% rename from testing/test_baseline rename to testing/baseline/test_baseline diff --git a/testing/test_baseline.py b/testing/baseline/test_baseline.py similarity index 100% rename from testing/test_baseline.py rename to testing/baseline/test_baseline.py diff --git a/testing/device_configs/tester1/device_config.json b/testing/device_configs/tester1/device_config.json new file mode 100644 index 000000000..268399b72 --- /dev/null +++ b/testing/device_configs/tester1/device_config.json @@ -0,0 +1,22 @@ +{ + "manufacturer": "Google", + "model": "Tester 1", + "mac_addr": "02:42:aa:00:00:01", + "test_modules": { + "dns": { + "enabled": false + }, + "connection": { + "enabled": false + }, + "ntp": { + "enabled": false + }, + "baseline": { + "enabled": false + }, + "nmap": { + "enabled": true + } + } +} diff --git a/testing/device_configs/tester2/device_config.json b/testing/device_configs/tester2/device_config.json new file mode 100644 index 000000000..12ac25811 --- /dev/null +++ b/testing/device_configs/tester2/device_config.json @@ -0,0 +1,22 @@ +{ + "manufacturer": "Google", + "model": "Tester 1", + "mac_addr": "02:42:aa:00:00:02", + "test_modules": { + "dns": { + "enabled": false + }, + "connection": { + "enabled": false + }, + "ntp": { + "enabled": true + }, + "baseline": { + "enabled": false + }, + "nmap": { + "enabled": true + } + } +} diff --git a/testing/test_pylint b/testing/pylint/test_pylint similarity index 100% rename from testing/test_pylint rename to testing/pylint/test_pylint diff --git a/testing/example/mac b/testing/tests/example/mac similarity index 100% rename from testing/example/mac rename to testing/tests/example/mac diff --git a/testing/example/mac1/results.json b/testing/tests/example/mac1/results.json similarity index 100% rename from testing/example/mac1/results.json rename to testing/tests/example/mac1/results.json diff --git a/testing/test_tests b/testing/tests/test_tests similarity index 95% rename from testing/test_tests rename to testing/tests/test_tests index c48e21e88..075507259 100755 --- a/testing/test_tests +++ b/testing/tests/test_tests @@ -50,6 +50,8 @@ cat <local/system.json } EOF +cp -r testing/device_configs local + sudo cmd/install TESTERS=$(jq -r 'keys[]' $MATRIX) @@ -61,14 +63,6 @@ for tester in $TESTERS; do ethmac=$(jq -r .$tester.ethmac $MATRIX) args=$(jq -r .$tester.args $MATRIX) - cat <local/devices/$tester/device_config.json - { - "manufacturer": "Google", - "model": "$args", - "mac_addr": "$ethmac" - } - EOF - touch $testrun_log sudo timeout 900 bin/testrun --single-intf --no-ui > $testrun_log 2>&1 & TPID=$! diff --git a/testing/test_tests.json b/testing/tests/test_tests.json similarity index 100% rename from testing/test_tests.json rename to testing/tests/test_tests.json diff --git a/testing/test_tests.py b/testing/tests/test_tests.py similarity index 100% rename from testing/test_tests.py rename to testing/tests/test_tests.py diff --git a/testing/unit_test/run_tests.sh b/testing/unit/run_tests.sh similarity index 100% rename from testing/unit_test/run_tests.sh rename to testing/unit/run_tests.sh From ec04823606eaf616a711fd98bbe89dc44dd958f5 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 14:15:56 +0100 Subject: [PATCH 18/32] Fix paths --- .../test/conn/python/src/connection_module.py | 8 +- modules/test/nmap/python/src/nmap_module.py | 79 ++++++++++--------- testing/baseline/test_baseline | 2 +- testing/tests/test_tests | 6 +- testing/tests/test_tests.py | 14 ++-- 5 files changed, 55 insertions(+), 54 deletions(-) diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index 74cf84735..d432d2131 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -131,10 +131,10 @@ def _connection_single_ip(self): LOGGER.info('Inspecting: ' + str(len(packets)) + ' packets') for packet in packets: # Option[1] = message-type, option 3 = DHCPREQUEST - if DHCP in packet and packet[DHCP].options[0][1] == 3: - mac_address = packet[Ether].src - if not mac_address.startswith(TR_CONTAINER_MAC_PREFIX): - mac_addresses.add(mac_address.upper()) + if DHCP in packet and packet[DHCP].options[0][1] == 3: + mac_address = packet[Ether].src + if not mac_address.startswith(TR_CONTAINER_MAC_PREFIX): + mac_addresses.add(mac_address.upper()) # Check if the device mac address is in the list of DHCPREQUESTs result = self._device_mac.upper() in mac_addresses diff --git a/modules/test/nmap/python/src/nmap_module.py b/modules/test/nmap/python/src/nmap_module.py index f998f302a..16db0b43a 100644 --- a/modules/test/nmap/python/src/nmap_module.py +++ b/modules/test/nmap/python/src/nmap_module.py @@ -92,7 +92,8 @@ def _process_port_results(self, tests): for test in tests: LOGGER.info("Checking scan results for test: " + str(test)) - self._check_scan_results(test_config=tests[test],scan_results=scan_results) + self._check_scan_results(test_config=tests[test], + scan_results=scan_results) def _check_unknown_ports(self,tests,scan_results): """ Check if any of the open ports detected are not defined @@ -103,54 +104,54 @@ def _check_unknown_ports(self,tests,scan_results): known_ports = [] for test in tests: if "tcp_ports" in tests[test]: - for port in tests[test]['tcp_ports']: + for port in tests[test]["tcp_ports"]: known_ports.append(port) if "udp_ports" in tests[test]: - for port in tests[test]['udp_ports']: + for port in tests[test]["udp_ports"]: known_ports.append(port) for port_result in scan_results: if not port_result in known_ports: LOGGER.info("Unknown port detected: " + port_result) - unallowed_port = {'port':port_result, - 'service':scan_results[port_result]['service'], - 'tcp_udp':scan_results[port_result]['tcp_udp']} + unallowed_port = {"port":port_result, + "service":scan_results[port_result]["service"], + "tcp_udp":scan_results[port_result]["tcp_udp"]} #self._unallowed_ports.append(unallowed_port) self._add_unknown_ports(tests,unallowed_port) def _add_unknown_ports(self,tests,unallowed_port): known_service = False - result = {'description':"Undefined port",'allowed':False} - if unallowed_port['tcp_udp'] == 'tcp': - port_style = 'tcp_ports' - elif unallowed_port['tcp_udp'] == 'udp': - port_style = 'udp_ports' + result = {"description":"Undefined port","allowed":False} + if unallowed_port["tcp_udp"] == "tcp": + port_style = "tcp_ports" + elif unallowed_port["tcp_udp"] == "udp": + port_style = "udp_ports" - LOGGER.info("Unknown Port Service: " + unallowed_port['service']) + LOGGER.info("Unknown Port Service: " + unallowed_port["service"]) for test in tests: LOGGER.debug("Checking for known service: " + test) # Create a regular expression pattern to match the variable at the # end of the string - port_service = r"\b" + re.escape(unallowed_port['service']) + r"\b$" + port_service = r"\b" + re.escape(unallowed_port["service"]) + r"\b$" service_match = re.search(port_service, test) if service_match: LOGGER.info("Service Matched: " + test) known_service=True for test_port in tests[test][port_style]: if "version" in tests[test][port_style][test_port]: - result['version'] = tests[test][port_style][test_port]['version'] + result["version"] = tests[test][port_style][test_port]["version"] if "description" in tests[test][port_style][test_port]: - result['description'] = tests[test][port_style][test_port]['description'] - result['inherited_from'] = test_port - if tests[test][port_style][test_port]['allowed']: - result['allowed'] = True + result["description"] = tests[test][port_style][test_port]["description"] + result["inherited_from"] = test_port + if tests[test][port_style][test_port]["allowed"]: + result["allowed"] = True break - tests[test][port_style][unallowed_port['port']]=result + tests[test][port_style][unallowed_port["port"]]=result break if not known_service: - service_name = "security.services.unknown." + str(unallowed_port['port']) - unknown_service = {port_style:{unallowed_port['port']:result}} + service_name = "security.services.unknown." + str(unallowed_port["port"]) + unknown_service = {port_style:{unallowed_port["port"]:result}} tests[service_name]=unknown_service @@ -176,7 +177,7 @@ def _check_scan_result(self,port_config,scan_results): self._unallowed_ports.append( {"port":str(port), "service":str(scan_results[port]["service"]), - 'tcp_udp':scan_results[port]['tcp_udp']} + "tcp_udp":scan_results[port]["tcp_udp"]} ) result = False else: @@ -208,39 +209,39 @@ def _check_unallowed_port(self,unallowed_ports,tests): version = None service = None for port in unallowed_ports: - LOGGER.info('Checking unallowed port: ' + port['port']) - LOGGER.info('Looking for service: ' + port['service']) - LOGGER.debug('Unallowed Port Config: ' + str(port)) - if port['tcp_udp'] == 'tcp': - port_style = 'tcp_ports' - elif port['tcp_udp'] == 'udp': - port_style = 'udp_ports' + LOGGER.info("Checking unallowed port: " + port["port"]) + LOGGER.info("Looking for service: " + port["service"]) + LOGGER.debug("Unallowed Port Config: " + str(port)) + if port["tcp_udp"] == "tcp": + port_style = "tcp_ports" + elif port["tcp_udp"] == "udp": + port_style = "udp_ports" for test in tests: - LOGGER.debug('Checking test: ' + str(test)) + LOGGER.debug("Checking test: " + str(test)) # Create a regular expression pattern to match the variable at the # end of the string - port_service = r"\b" + re.escape(port['service']) + r"\b$" + port_service = r"\b" + re.escape(port["service"]) + r"\b$" service_match = re.search(port_service, test) if service_match: LOGGER.info("Service Matched: " + test) service_config = tests[test] - service = port['service'] + service = port["service"] for service_port in service_config[port_style]: port_config = service_config[port_style][service_port] - service_allowed |= port_config['allowed'] - version = port_config['version'] if 'version' in port_config else None + service_allowed |= port_config["allowed"] + version = port_config["version"] if "version" in port_config else None if service_allowed: LOGGER.info("Unallowed port detected for allowed service: " + service) if version is not None: allowed = self._check_version(service=service, - version_detected=self._scan_tcp_results[port['port']]['version'], + version_detected=self._scan_tcp_results[port["port"]]["version"], version_expected=version) else: allowed = True if allowed: - LOGGER.info("Unallowed port exception for approved service: " + port['port']) + LOGGER.info("Unallowed port exception for approved service: " + port["port"]) for u_port in self._unallowed_ports: - if port['port'] in u_port['port']: + if port["port"] in u_port["port"]: self._unallowed_ports.remove(u_port) break break @@ -318,7 +319,7 @@ def _scan_udp_with_script(self, script_name, ports=None): nmap_results_json = self._nmap_results_to_json(nmap_results) return self._process_nmap_json_results(nmap_results_json=nmap_results_json) - def _scan_tcp_ports(self, tests): + def _scan_tcp_ports(self): max_port = 65535 LOGGER.info("Running nmap TCP port scan") nmap_results = util.run_command( @@ -382,4 +383,4 @@ def _json_port_to_dict(self,port_json): if "@extrainfo" in port_json["service"]: port["version"] += " " + port_json["service"]["@extrainfo"] port_result = {port_json["@portid"]:port} - return port_result \ No newline at end of file + return port_result diff --git a/testing/baseline/test_baseline b/testing/baseline/test_baseline index 9aec17374..61d0f9b56 100755 --- a/testing/baseline/test_baseline +++ b/testing/baseline/test_baseline @@ -80,6 +80,6 @@ echo "Done baseline test" more $TESTRUN_OUT -pytest testing/test_baseline.py +pytest testing/baseline/test_baseline.py exit $? \ No newline at end of file diff --git a/testing/tests/test_tests b/testing/tests/test_tests index 075507259..d2588d3f0 100755 --- a/testing/tests/test_tests +++ b/testing/tests/test_tests @@ -17,7 +17,7 @@ set -o xtrace ip a TEST_DIR=/tmp/results -MATRIX=testing/test_tests.json +MATRIX=testing/tests/test_tests.json mkdir -p $TEST_DIR @@ -50,7 +50,7 @@ cat <local/system.json } EOF -cp -r testing/device_configs local +cp -r testing/device_configs/* local sudo cmd/install @@ -117,6 +117,6 @@ for tester in $TESTERS; do done -pytest -v testing/test_tests.py +pytest -v testing/tests/test_tests.py exit $? diff --git a/testing/tests/test_tests.py b/testing/tests/test_tests.py index b61fdf064..1f484647a 100644 --- a/testing/tests/test_tests.py +++ b/testing/tests/test_tests.py @@ -29,7 +29,7 @@ TEST_MATRIX = 'test_tests.json' RESULTS_PATH = '/tmp/results/*.json' -#TODO add reason +#TODO add reason @dataclass(frozen=True) class TestResult: name: str @@ -80,14 +80,14 @@ def test_list_tests(capsys, results, test_matrix): all_tests = set(itertools.chain.from_iterable( [collect_actual_results(results[x]) for x in results.keys()])) - ci_pass = set([test - for testers in test_matrix.values() - for test, result in testers['expected_results'].items() + ci_pass = set([test + for testers in test_matrix.values() + for test, result in testers['expected_results'].items() if result == 'compliant']) - ci_fail = set([test - for testers in test_matrix.values() - for test, result in testers['expected_results'].items() + ci_fail = set([test + for testers in test_matrix.values() + for test, result in testers['expected_results'].items() if result == 'non-compliant']) with capsys.disabled(): From 9169adfb8cfc1d299cbf41a5e0a4e259bb7935f4 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 14:34:07 +0100 Subject: [PATCH 19/32] Change MAC address --- testing/device_configs/tester2/device_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/device_configs/tester2/device_config.json b/testing/device_configs/tester2/device_config.json index 12ac25811..c6b82692e 100644 --- a/testing/device_configs/tester2/device_config.json +++ b/testing/device_configs/tester2/device_config.json @@ -1,7 +1,7 @@ { "manufacturer": "Google", "model": "Tester 1", - "mac_addr": "02:42:aa:00:00:02", + "mac_addr": "02:42:ac:12:00:02", "test_modules": { "dns": { "enabled": false From b9fad2ce882f8cb05a43a20f17b78a569b79dc0c Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 15:26:11 +0100 Subject: [PATCH 20/32] Revert mac --- testing/device_configs/tester2/device_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/device_configs/tester2/device_config.json b/testing/device_configs/tester2/device_config.json index c6b82692e..8b090d80a 100644 --- a/testing/device_configs/tester2/device_config.json +++ b/testing/device_configs/tester2/device_config.json @@ -1,7 +1,7 @@ { "manufacturer": "Google", - "model": "Tester 1", - "mac_addr": "02:42:ac:12:00:02", + "model": "Tester 2", + "mac_addr": "02:42:aa:00:00:02", "test_modules": { "dns": { "enabled": false From 215b55c77fa53326d8ebc408503a2cfc77a8936e Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 15:46:38 +0100 Subject: [PATCH 21/32] Fix copy path --- testing/tests/test_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/test_tests b/testing/tests/test_tests index d2588d3f0..50339d247 100755 --- a/testing/tests/test_tests +++ b/testing/tests/test_tests @@ -50,7 +50,7 @@ cat <local/system.json } EOF -cp -r testing/device_configs/* local +cp -r testing/device_configs/* local/devices sudo cmd/install From b09514c71ea4e26c4a63a38988d7cecd3799a2a1 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 16:25:20 +0100 Subject: [PATCH 22/32] Debug loading devices --- framework/python/src/core/testrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index a66c2c859..ca9fca775 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -74,7 +74,6 @@ def __init__(self, else: self._config_file = self._get_config_abs(config_file) - self._devices = [] self._net_only = net_only self._single_intf = single_intf self._no_ui = no_ui @@ -129,6 +128,7 @@ def _load_devices(self, device_dir): mac_addr=mac_addr, test_modules=test_modules) self.get_session().add_device(device) + LOGGER.debug(f'Loaded device {device.manufacturer} {device.model} with MAC address {device.mac_addr}') def start(self): From 9d6eaaf1e46288f03564e4478749db48ce4764ad Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 19 Jul 2023 16:32:15 +0100 Subject: [PATCH 23/32] Remove reference --- framework/python/src/core/testrun.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index ca9fca775..f8717f2c0 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -105,7 +105,7 @@ def __init__(self, def _load_all_devices(self): self._load_devices(device_dir=LOCAL_DEVICES_DIR) self._load_devices(device_dir=RESOURCE_DEVICES_DIR) - return self._devices + return self.get_session().get_device_repository() def _load_devices(self, device_dir): LOGGER.debug('Loading devices from ' + device_dir) @@ -233,9 +233,10 @@ def _stop_tests(self): def get_device(self, mac_addr): """Returns a loaded device object from the device mac address.""" - for device in self._devices: + for device in self._session.get_device_repository(): if device.mac_addr == mac_addr: return device + return None def _device_discovered(self, mac_addr): From ef3dd3c8dc673ee129ca501a0a7216791971b028 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 12:08:44 +0100 Subject: [PATCH 24/32] Changes --- bin/testrun | 3 +++ framework/python/src/api/api.py | 31 +++++++++++++++++--------- framework/python/src/common/session.py | 21 ++++++----------- framework/python/src/core/testrun.py | 10 ++++----- testing/tests/test_tests | 1 + 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/bin/testrun b/bin/testrun index 13a12010c..9281c1ac6 100755 --- a/bin/testrun +++ b/bin/testrun @@ -26,6 +26,9 @@ fi # Ensure that /var/run/netns folder exists sudo mkdir -p /var/run/netns +# Create device folder if it doesn't exist +mkdir -p local/devices + # Check if Python modules exist. Install if not [ ! -d "venv" ] && sudo cmd/install diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 1873393bd..cd88617cd 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -71,10 +71,15 @@ async def get_sys_interfaces(self): ifaces.append(iface) return ifaces - async def post_sys_config(self, request: Request): - config = (await request.body()).decode("UTF-8") - config_json = json.loads(config) - self._session.set_config(config_json) + async def post_sys_config(self, request: Request, response: Response): + try: + config = (await request.body()).decode("UTF-8") + config_json = json.loads(config) + self._session.set_config(config_json) + # Catch JSON Decode error etc + except JSONDecodeError: + response.status_code = status.HTTP_400_BAD_REQUEST + return self._generate_msg(False, "Invalid JSON received") return self._session.get_config() async def get_sys_config(self): @@ -95,11 +100,11 @@ async def start_test_run(self, request: Request, response: Response): body_json = json.loads(body) except JSONDecodeError: response.status_code = status.HTTP_400_BAD_REQUEST - return json.loads('{"error": "Invalid JSON received"}') + return self._generate_msg(False, "Invalid JSON received") if "device" not in body_json or "mac_addr" not in body_json["device"]: response.status_code = status.HTTP_400_BAD_REQUEST - return json.loads('{"error": "Valid device MAC address has not been specified"}') + return self._generate_msg(False, "Invalid request received") device = self._session.get_device(body_json["device"]["mac_addr"]) @@ -107,17 +112,17 @@ async def start_test_run(self, request: Request, response: Response): if self._test_run.get_session().get_status() != "Idle": LOGGER.debug("Test Run is already running. Cannot start another instance") response.status_code = status.HTTP_409_CONFLICT - return json.loads('{"error": "Test Run is already running"}') + return self._generate_msg(False, "Test Run is already running") # Check if requested device is known in the device repository if device is None: response.status_code = status.HTTP_404_NOT_FOUND - return json.loads('{"error": "A device with that MAC address could not be found"}') + return self._generate_msg(False, "A device with that MAC address could not be found") # Check Test Run is able to start if self._test_run.get_net_orc().check_config() is False: response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR - return json.loads('{"error": "Configured interfaces are not ready for use. Ensure both interfaces are connected."}') + return self._generate_msg(False, "Configured interfaces are not ready for use. Ensure both interfaces are connected.") self._test_run.get_session().set_target_device(device) LOGGER.info(f"Starting Test Run with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}") @@ -127,13 +132,19 @@ async def start_test_run(self, request: Request, response: Response): thread.start() return self._test_run.get_session().to_json() + def _generate_msg(self, success, message): + msg_type = "success" + if not success: + msg_type = "error" + return json.loads('{"' + msg_type + '": "' + message + '"}') + def _start_test_run(self): self._test_run.start() async def stop_test_run(self): LOGGER.info("Received stop command. Stopping Test Run") self._test_run.stop() - return json.loads('{"success": "Test Run stopped"}') + return self._generate_msg(True, "Test Run stopped") async def get_status(self): return self._test_run.get_session().to_json() diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 91abd5a26..9b695f7dc 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -77,23 +77,16 @@ def _load_config(self): config_file_json = json.load(f) # Network interfaces - if NETWORK_KEY in config_file_json: - if DEVICE_INTF_KEY in config_file_json[NETWORK_KEY]: - self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json[NETWORK_KEY][DEVICE_INTF_KEY] - if INTERNET_INTF_KEY in config_file_json[NETWORK_KEY]: - self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json[NETWORK_KEY][INTERNET_INTF_KEY] + self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) + self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY) - if RUNTIME_KEY in config_file_json: - self._config[RUNTIME_KEY] = config_file_json[RUNTIME_KEY] + self._config[RUNTIME_KEY] = config_file_json.get(RUNTIME_KEY) - if STARTUP_TIMEOUT_KEY in config_file_json: - self._config[STARTUP_TIMEOUT_KEY] = config_file_json[STARTUP_TIMEOUT_KEY] + self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(STARTUP_TIMEOUT_KEY) - if MONITOR_PERIOD_KEY in config_file_json: - self._config[MONITOR_PERIOD_KEY] = config_file_json[MONITOR_PERIOD_KEY] + self._config[MONITOR_PERIOD_KEY] = config_file_json.get(MONITOR_PERIOD_KEY) - if LOG_LEVEL_KEY in config_file_json: - self._config[LOG_LEVEL_KEY] = config_file_json[LOG_LEVEL_KEY] + self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY) def _save_config(self): with open(self._config_file, 'w', encoding='utf-8') as f: @@ -113,7 +106,7 @@ def get_internet_interface(self): def get_monitor_period(self): return self._config[MONITOR_PERIOD_KEY] - + def get_startup_timeout(self): return self._config[STARTUP_TIMEOUT_KEY] diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index f8717f2c0..6016fbfe7 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -28,6 +28,10 @@ from common import logger, util from common.device import Device from common.session import TestRunSession +from api.api import Api +from net_orc.listener import NetworkEvent +from net_orc import network_orchestrator as net_orc +from test_orc import test_orchestrator as test_orc # Locate parent directory current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -36,11 +40,6 @@ root_dir = os.path.dirname(os.path.dirname( os.path.dirname(os.path.dirname(current_dir)))) -from api.api import Api # pylint: disable=wrong-import-position -from net_orc.listener import NetworkEvent # pylint: disable=wrong-import-position -from test_orc import test_orchestrator as test_orc # pylint: disable=wrong-import-position -from net_orc import network_orchestrator as net_orc # pylint: disable=wrong-import-position - LOGGER = logger.get_logger('test_run') DEFAULT_CONFIG_FILE = 'local/system.json' @@ -110,7 +109,6 @@ def _load_all_devices(self): def _load_devices(self, device_dir): LOGGER.debug('Loading devices from ' + device_dir) - os.makedirs(device_dir, exist_ok=True) util.run_command(f'chown -R {util.get_host_user()} {device_dir}') for device_folder in os.listdir(device_dir): diff --git a/testing/tests/test_tests b/testing/tests/test_tests index 50339d247..be7a3cef3 100755 --- a/testing/tests/test_tests +++ b/testing/tests/test_tests @@ -50,6 +50,7 @@ cat <local/system.json } EOF +mkdir -p local/devices cp -r testing/device_configs/* local/devices sudo cmd/install From 6c38fafb3fd10e8ffc542b417065fdb6e8aff082 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 12:59:30 +0100 Subject: [PATCH 25/32] Re-add checks to prevent null values --- framework/python/src/common/session.py | 31 ++++++++++++++++---------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 9b695f7dc..8738fcfdc 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -77,38 +77,45 @@ def _load_config(self): config_file_json = json.load(f) # Network interfaces - self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) - self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY) + if (NETWORK_KEY in self._config_file_json + and DEVICE_INTF_KEY in config_file_json.get(NETWORK_KEY) + and INTERNET_INTF_KEY in config_file_json(NETWORK_KEY)): + self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) + self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY) - self._config[RUNTIME_KEY] = config_file_json.get(RUNTIME_KEY) + if RUNTIME_KEY in config_file_json: + self._config[RUNTIME_KEY] = config_file_json.get(RUNTIME_KEY) - self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(STARTUP_TIMEOUT_KEY) + if STARTUP_TIMEOUT_KEY in config_file_json: + self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(STARTUP_TIMEOUT_KEY) - self._config[MONITOR_PERIOD_KEY] = config_file_json.get(MONITOR_PERIOD_KEY) + if MONITOR_PERIOD_KEY in config_file_json: + self._config[MONITOR_PERIOD_KEY] = config_file_json.get(MONITOR_PERIOD_KEY) - self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY) + if LOG_LEVEL_KEY in config_file_json: + self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY) def _save_config(self): with open(self._config_file, 'w', encoding='utf-8') as f: f.write(json.dumps(self._config, indent=2)) def get_runtime(self): - return self._config[RUNTIME_KEY] + return self._config.get(RUNTIME_KEY) def get_log_level(self): - return self._config[LOG_LEVEL_KEY] + return self._config.get(LOG_LEVEL_KEY) def get_device_interface(self): - return self._config[NETWORK_KEY][DEVICE_INTF_KEY] + return self._config.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) def get_internet_interface(self): - return self._config[NETWORK_KEY][INTERNET_INTF_KEY] + return self._config.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY) def get_monitor_period(self): - return self._config[MONITOR_PERIOD_KEY] + return self._config.get(MONITOR_PERIOD_KEY) def get_startup_timeout(self): - return self._config[STARTUP_TIMEOUT_KEY] + return self._config.get(STARTUP_TIMEOUT_KEY) def set_config(self, config_json): self._config = config_json From f14ffebbaceb4e0deb684fd7278a342d10047dd2 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 13:06:11 +0100 Subject: [PATCH 26/32] Fix variable --- framework/python/src/common/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 8738fcfdc..11da2d854 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -77,7 +77,7 @@ def _load_config(self): config_file_json = json.load(f) # Network interfaces - if (NETWORK_KEY in self._config_file_json + if (NETWORK_KEY in config_file_json and DEVICE_INTF_KEY in config_file_json.get(NETWORK_KEY) and INTERNET_INTF_KEY in config_file_json(NETWORK_KEY)): self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) From 975a8943d43faa3570f11d6fe0a59d1db738ed4c Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 13:07:57 +0100 Subject: [PATCH 27/32] Fix --- framework/python/src/common/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 11da2d854..74e160ac5 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -79,7 +79,7 @@ def _load_config(self): # Network interfaces if (NETWORK_KEY in config_file_json and DEVICE_INTF_KEY in config_file_json.get(NETWORK_KEY) - and INTERNET_INTF_KEY in config_file_json(NETWORK_KEY)): + and INTERNET_INTF_KEY in config_file_json.get(NETWORK_KEY)): self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY) self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY) From bd99c5e7eb3397758f3deb030efffb66e853f1b1 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 13:47:44 +0100 Subject: [PATCH 28/32] Use dict instead of string --- framework/python/src/test_orc/test_orchestrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 03bcfd0bb..1da27407b 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -117,7 +117,7 @@ def test_in_progress(self): def _is_module_enabled(self, module, device): enabled = True if device.test_modules is not None: - test_modules = json.loads(device.test_modules) + test_modules = json.dumps(device.test_modules) if module.name in test_modules: if "enabled" in test_modules[module.name]: enabled = test_modules[module.name]["enabled"] From 3908cde9e9d9d92667f0de8860660a5ebb6d85b1 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 14:40:18 +0100 Subject: [PATCH 29/32] Try without json conversion --- framework/python/src/test_orc/test_orchestrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 1da27407b..424abc888 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -117,7 +117,7 @@ def test_in_progress(self): def _is_module_enabled(self, module, device): enabled = True if device.test_modules is not None: - test_modules = json.dumps(device.test_modules) + test_modules = device.test_modules if module.name in test_modules: if "enabled" in test_modules[module.name]: enabled = test_modules[module.name]["enabled"] From 1b286a7563a1fa9013159c2d2dd19e3cf6741c29 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 17:10:58 +0100 Subject: [PATCH 30/32] Container output to log --- .../python/src/test_orc/test_orchestrator.py | 10 +++++++--- modules/test/base/base.Dockerfile | 2 +- modules/test/base/bin/capture | 2 +- modules/test/base/bin/setup_binaries | 2 +- modules/test/base/bin/start | 17 +++++++++++++++++ modules/test/base/bin/start_module | 2 +- modules/test/conn/bin/start_test_module | 2 +- modules/test/dns/python/src/dns_module.py | 3 +-- 8 files changed, 30 insertions(+), 10 deletions(-) create mode 100755 modules/test/base/bin/start diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 424abc888..646f5cf4b 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -182,7 +182,7 @@ def _run_test_module(self, module, device): environment={ "HOST_USER": self._host_user, "DEVICE_MAC": device.mac_addr, - "DEVICE_TEST_MODULES": device.test_modules, + "DEVICE_TEST_MODULES": json.dumps(device.test_modules), "IPV4_SUBNET": self._net_orc.network_config.ipv4_network, "IPV6_SUBNET": self._net_orc.network_config.ipv6_network }) @@ -201,8 +201,12 @@ def _run_test_module(self, module, device): test_module_timeout = time.time() + module.timeout status = self._get_module_status(module) - while time.time() < test_module_timeout and status == "running": - time.sleep(1) + log_stream = module.container.logs(stream=True, stdout=True, stderr=True) + while (time.time() < test_module_timeout and + status == "running" and + self._session.get_status() == "In progress"): + line = next(log_stream).decode("utf-8").strip() + print(line) status = self._get_module_status(module) LOGGER.info("Test module " + module.name + " has finished") diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index 10344cbc7..62ff54d6c 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -45,4 +45,4 @@ COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ # Start the test module -ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file +ENTRYPOINT [ "/testrun/bin/start" ] \ No newline at end of file diff --git a/modules/test/base/bin/capture b/modules/test/base/bin/capture index 69fa916c3..a4deb988a 100644 --- a/modules/test/base/bin/capture +++ b/modules/test/base/bin/capture @@ -27,7 +27,7 @@ INTERFACE=$2 # Create the output directory and start the capture mkdir -p $PCAP_DIR chown $HOST_USER $PCAP_DIR -tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & +tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & > /dev/null 2>&1 # Small pause to let the capture to start sleep 1 \ No newline at end of file diff --git a/modules/test/base/bin/setup_binaries b/modules/test/base/bin/setup_binaries index 6af744693..eaccf9de6 100644 --- a/modules/test/base/bin/setup_binaries +++ b/modules/test/base/bin/setup_binaries @@ -18,7 +18,7 @@ BIN_DIR=$1 # Remove incorrect line endings -dos2unix $BIN_DIR/* +dos2unix $BIN_DIR/* >/dev/null 2>&1 # Make sure all the bin files are executable chmod u+x $BIN_DIR/* \ No newline at end of file diff --git a/modules/test/base/bin/start b/modules/test/base/bin/start new file mode 100755 index 000000000..6869d1116 --- /dev/null +++ b/modules/test/base/bin/start @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/testrun/bin/start_module > /dev/null \ No newline at end of file diff --git a/modules/test/base/bin/start_module b/modules/test/base/bin/start_module index 82c9d26bf..69f399feb 100644 --- a/modules/test/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -99,4 +99,4 @@ fi sleep 3 # Start the networking service -$BIN_DIR/start_test_module $MODULE_NAME $IFACE \ No newline at end of file +$BIN_DIR/start_test_module $MODULE_NAME $IFACE > /runtime/output/container.log \ No newline at end of file diff --git a/modules/test/conn/bin/start_test_module b/modules/test/conn/bin/start_test_module index 0df510b86..d85ae7d6b 100644 --- a/modules/test/conn/bin/start_test_module +++ b/modules/test/conn/bin/start_test_module @@ -45,7 +45,7 @@ touch $RESULT_FILE chown $HOST_USER $LOG_FILE chown $HOST_USER $RESULT_FILE -# Run the python scrip that will execute the tests for this module +# Run the python script that will execute the tests for this module # -u flag allows python print statements # to be logged by docker by running unbuffered python3 -u $PYTHON_SRC_DIR/run.py "-m $MODULE_NAME" diff --git a/modules/test/dns/python/src/dns_module.py b/modules/test/dns/python/src/dns_module.py index 8d32d4dfb..aecbd5bd1 100644 --- a/modules/test/dns/python/src/dns_module.py +++ b/modules/test/dns/python/src/dns_module.py @@ -55,8 +55,7 @@ def _dns_network_from_dhcp(self): self._dns_server) # Check if the device DNS traffic is to appropriate server - tcpdump_filter = (f'dst port 53 and dst host {self._dns_server}', - f' and ether src {self._device_mac}') + tcpdump_filter = f'dst port 53 and dst host {self._dns_server} and ether src {self._device_mac}' result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) From aa9f03ff7522d63d3d5e37a7ebc32d296ed9aaa5 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 20 Jul 2023 18:03:01 +0100 Subject: [PATCH 31/32] Undo changes to nmap module --- modules/test/base/bin/capture | 2 +- modules/test/nmap/python/src/nmap_module.py | 79 ++++++++++----------- 2 files changed, 40 insertions(+), 41 deletions(-) diff --git a/modules/test/base/bin/capture b/modules/test/base/bin/capture index a4deb988a..e237f3d72 100644 --- a/modules/test/base/bin/capture +++ b/modules/test/base/bin/capture @@ -27,7 +27,7 @@ INTERFACE=$2 # Create the output directory and start the capture mkdir -p $PCAP_DIR chown $HOST_USER $PCAP_DIR -tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & > /dev/null 2>&1 +tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & 2>&1 /dev/null # Small pause to let the capture to start sleep 1 \ No newline at end of file diff --git a/modules/test/nmap/python/src/nmap_module.py b/modules/test/nmap/python/src/nmap_module.py index 16db0b43a..f998f302a 100644 --- a/modules/test/nmap/python/src/nmap_module.py +++ b/modules/test/nmap/python/src/nmap_module.py @@ -92,8 +92,7 @@ def _process_port_results(self, tests): for test in tests: LOGGER.info("Checking scan results for test: " + str(test)) - self._check_scan_results(test_config=tests[test], - scan_results=scan_results) + self._check_scan_results(test_config=tests[test],scan_results=scan_results) def _check_unknown_ports(self,tests,scan_results): """ Check if any of the open ports detected are not defined @@ -104,54 +103,54 @@ def _check_unknown_ports(self,tests,scan_results): known_ports = [] for test in tests: if "tcp_ports" in tests[test]: - for port in tests[test]["tcp_ports"]: + for port in tests[test]['tcp_ports']: known_ports.append(port) if "udp_ports" in tests[test]: - for port in tests[test]["udp_ports"]: + for port in tests[test]['udp_ports']: known_ports.append(port) for port_result in scan_results: if not port_result in known_ports: LOGGER.info("Unknown port detected: " + port_result) - unallowed_port = {"port":port_result, - "service":scan_results[port_result]["service"], - "tcp_udp":scan_results[port_result]["tcp_udp"]} + unallowed_port = {'port':port_result, + 'service':scan_results[port_result]['service'], + 'tcp_udp':scan_results[port_result]['tcp_udp']} #self._unallowed_ports.append(unallowed_port) self._add_unknown_ports(tests,unallowed_port) def _add_unknown_ports(self,tests,unallowed_port): known_service = False - result = {"description":"Undefined port","allowed":False} - if unallowed_port["tcp_udp"] == "tcp": - port_style = "tcp_ports" - elif unallowed_port["tcp_udp"] == "udp": - port_style = "udp_ports" + result = {'description':"Undefined port",'allowed':False} + if unallowed_port['tcp_udp'] == 'tcp': + port_style = 'tcp_ports' + elif unallowed_port['tcp_udp'] == 'udp': + port_style = 'udp_ports' - LOGGER.info("Unknown Port Service: " + unallowed_port["service"]) + LOGGER.info("Unknown Port Service: " + unallowed_port['service']) for test in tests: LOGGER.debug("Checking for known service: " + test) # Create a regular expression pattern to match the variable at the # end of the string - port_service = r"\b" + re.escape(unallowed_port["service"]) + r"\b$" + port_service = r"\b" + re.escape(unallowed_port['service']) + r"\b$" service_match = re.search(port_service, test) if service_match: LOGGER.info("Service Matched: " + test) known_service=True for test_port in tests[test][port_style]: if "version" in tests[test][port_style][test_port]: - result["version"] = tests[test][port_style][test_port]["version"] + result['version'] = tests[test][port_style][test_port]['version'] if "description" in tests[test][port_style][test_port]: - result["description"] = tests[test][port_style][test_port]["description"] - result["inherited_from"] = test_port - if tests[test][port_style][test_port]["allowed"]: - result["allowed"] = True + result['description'] = tests[test][port_style][test_port]['description'] + result['inherited_from'] = test_port + if tests[test][port_style][test_port]['allowed']: + result['allowed'] = True break - tests[test][port_style][unallowed_port["port"]]=result + tests[test][port_style][unallowed_port['port']]=result break if not known_service: - service_name = "security.services.unknown." + str(unallowed_port["port"]) - unknown_service = {port_style:{unallowed_port["port"]:result}} + service_name = "security.services.unknown." + str(unallowed_port['port']) + unknown_service = {port_style:{unallowed_port['port']:result}} tests[service_name]=unknown_service @@ -177,7 +176,7 @@ def _check_scan_result(self,port_config,scan_results): self._unallowed_ports.append( {"port":str(port), "service":str(scan_results[port]["service"]), - "tcp_udp":scan_results[port]["tcp_udp"]} + 'tcp_udp':scan_results[port]['tcp_udp']} ) result = False else: @@ -209,39 +208,39 @@ def _check_unallowed_port(self,unallowed_ports,tests): version = None service = None for port in unallowed_ports: - LOGGER.info("Checking unallowed port: " + port["port"]) - LOGGER.info("Looking for service: " + port["service"]) - LOGGER.debug("Unallowed Port Config: " + str(port)) - if port["tcp_udp"] == "tcp": - port_style = "tcp_ports" - elif port["tcp_udp"] == "udp": - port_style = "udp_ports" + LOGGER.info('Checking unallowed port: ' + port['port']) + LOGGER.info('Looking for service: ' + port['service']) + LOGGER.debug('Unallowed Port Config: ' + str(port)) + if port['tcp_udp'] == 'tcp': + port_style = 'tcp_ports' + elif port['tcp_udp'] == 'udp': + port_style = 'udp_ports' for test in tests: - LOGGER.debug("Checking test: " + str(test)) + LOGGER.debug('Checking test: ' + str(test)) # Create a regular expression pattern to match the variable at the # end of the string - port_service = r"\b" + re.escape(port["service"]) + r"\b$" + port_service = r"\b" + re.escape(port['service']) + r"\b$" service_match = re.search(port_service, test) if service_match: LOGGER.info("Service Matched: " + test) service_config = tests[test] - service = port["service"] + service = port['service'] for service_port in service_config[port_style]: port_config = service_config[port_style][service_port] - service_allowed |= port_config["allowed"] - version = port_config["version"] if "version" in port_config else None + service_allowed |= port_config['allowed'] + version = port_config['version'] if 'version' in port_config else None if service_allowed: LOGGER.info("Unallowed port detected for allowed service: " + service) if version is not None: allowed = self._check_version(service=service, - version_detected=self._scan_tcp_results[port["port"]]["version"], + version_detected=self._scan_tcp_results[port['port']]['version'], version_expected=version) else: allowed = True if allowed: - LOGGER.info("Unallowed port exception for approved service: " + port["port"]) + LOGGER.info("Unallowed port exception for approved service: " + port['port']) for u_port in self._unallowed_ports: - if port["port"] in u_port["port"]: + if port['port'] in u_port['port']: self._unallowed_ports.remove(u_port) break break @@ -319,7 +318,7 @@ def _scan_udp_with_script(self, script_name, ports=None): nmap_results_json = self._nmap_results_to_json(nmap_results) return self._process_nmap_json_results(nmap_results_json=nmap_results_json) - def _scan_tcp_ports(self): + def _scan_tcp_ports(self, tests): max_port = 65535 LOGGER.info("Running nmap TCP port scan") nmap_results = util.run_command( @@ -383,4 +382,4 @@ def _json_port_to_dict(self,port_json): if "@extrainfo" in port_json["service"]: port["version"] += " " + port_json["service"]["@extrainfo"] port_result = {port_json["@portid"]:port} - return port_result + return port_result \ No newline at end of file From cd855daa51ff6212fad040a8d23d6c88e3a2e993 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Fri, 21 Jul 2023 22:13:46 +0100 Subject: [PATCH 32/32] Add post devices route --- framework/python/src/api/api.py | 46 ++++++++++++++++++- framework/python/src/common/session.py | 4 ++ .../python/src/test_orc/test_orchestrator.py | 7 ++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index cd88617cd..d877a5b33 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -20,9 +20,14 @@ import uvicorn from common import logger +from common.device import Device LOGGER = logger.get_logger("api") +DEVICE_MAC_ADDR_KEY = "mac_addr" +DEVICE_MANUFACTURER_KEY = "manufacturer" +DEVICE_MODEL_KEY = "model" + class Api: """Provide REST endpoints to manage Test Run""" @@ -45,6 +50,7 @@ def __init__(self, test_run): self._router.add_api_route("/system/status", self.get_status) self._router.add_api_route("/devices", self.get_devices) + self._router.add_api_route("/device", self.save_device, methods=["POST"]) self._app = FastAPI() self._app.include_router(self._router) @@ -142,7 +148,7 @@ def _start_test_run(self): self._test_run.start() async def stop_test_run(self): - LOGGER.info("Received stop command. Stopping Test Run") + LOGGER.debug("Received stop command. Stopping Test Run") self._test_run.stop() return self._generate_msg(True, "Test Run stopped") @@ -151,3 +157,41 @@ async def get_status(self): async def get_history(self): LOGGER.info("Returning previous Test Runs to UI") + + async def save_device(self, request: Request, response: Response): + LOGGER.debug("Received device post request") + + try: + device_raw = (await request.body()).decode("UTF-8") + device_json = json.loads(device_raw) + + if not self._validate_device_json(device_json): + response.status_code = status.HTTP_400_BAD_REQUEST + return self._generate_msg(False, "Invalid request received") + + device = self._session.get_device(device_json.get(DEVICE_MAC_ADDR_KEY)) + if device is None: + # Create new device + device = Device() + device.mac_addr = device_json.get(DEVICE_MAC_ADDR_KEY) + response.status_code = status.HTTP_201_CREATED + + device.manufacturer = device_json.get(DEVICE_MANUFACTURER_KEY) + device.model = device_json.get(DEVICE_MODEL_KEY) + + self._session.save_device(device) + + return device + + # Catch JSON Decode error etc + except JSONDecodeError: + response.status_code = status.HTTP_400_BAD_REQUEST + return self._generate_msg(False, "Invalid JSON received") + + def _validate_device_json(self, json_obj): + if not (DEVICE_MAC_ADDR_KEY in json_obj and + DEVICE_MANUFACTURER_KEY in json_obj and + DEVICE_MODEL_KEY in json_obj + ): + return False + return True diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 74e160ac5..a0f6118ff 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -139,6 +139,10 @@ def get_device(self, mac_addr): return device return None + def save_device(self, device): + # TODO: We need to save the folder path of the device config + return + def get_status(self): return self._status diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 646f5cf4b..74e399df1 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -205,8 +205,11 @@ def _run_test_module(self, module, device): while (time.time() < test_module_timeout and status == "running" and self._session.get_status() == "In progress"): - line = next(log_stream).decode("utf-8").strip() - print(line) + try: + line = next(log_stream).decode("utf-8").strip() + print(line) + except Exception: + time.sleep(1) status = self._get_module_status(module) LOGGER.info("Test module " + module.name + " has finished")