diff --git a/cmd/install b/cmd/install index 23e463158..f5af3a5d3 100755 --- a/cmd/install +++ b/cmd/install @@ -4,6 +4,8 @@ python3 -m venv venv source venv/bin/activate +pip3 install --upgrade requests + pip3 install -r framework/requirements.txt pip3 install -r net_orc/python/requirements.txt diff --git a/framework/device.py b/framework/device.py index c17dd8e3a..74d62d495 100644 --- a/framework/device.py +++ b/framework/device.py @@ -1,12 +1,14 @@ -"""Track device object information.""" -from dataclasses import dataclass -from network_device import NetworkDevice - - -@dataclass -class Device(NetworkDevice): - """Represents a physical device and it's configuration.""" - - make: str = None - model: str = None - test_modules: str = None +"""Track device object information.""" + +from network_device import NetworkDevice +from dataclasses import dataclass + + +@dataclass +class Device(NetworkDevice): + """Represents a physical device and it's configuration.""" + + make: str = None + model: str = None + mac_addr: str + test_modules: str = None diff --git a/framework/testrun.py b/framework/testrun.py index b9cb6a0e5..44c3bca6d 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -46,142 +46,142 @@ class TestRun: # pylint: disable=too-few-public-methods - """Test Run controller. + """Test Run controller. - Creates an instance of the network orchestrator, test - orchestrator and user interface. - """ + Creates an instance of the network orchestrator, test + orchestrator and user interface. + """ - def __init__(self, config_file=CONFIG_FILE, validate=True, net_only=False, single_intf=False): - self._devices = [] - self._net_only = net_only - self._single_intf = single_intf + def __init__(self, config_file=CONFIG_FILE, validate=True, net_only=False, single_intf=False): + self._devices = [] + self._net_only = net_only + self._single_intf = single_intf - # Catch any exit signals - self._register_exits() + # Catch any exit signals + self._register_exits() - # Expand the config file to absolute pathing - config_file_abs = self._get_config_abs(config_file=config_file) + # Expand the config file to absolute pathing + config_file_abs = self._get_config_abs(config_file=config_file) - self._net_orc = net_orc.NetworkOrchestrator( - config_file=config_file_abs, - validate=validate, - async_monitor=not self._net_only, - single_intf = self._single_intf) - self._test_orc = test_orc.TestOrchestrator() + self._net_orc = net_orc.NetworkOrchestrator( + config_file=config_file_abs, + validate=validate, + async_monitor=not self._net_only, + single_intf = self._single_intf) + self._test_orc = test_orc.TestOrchestrator(self._net_orc) - def start(self): + def start(self): - self._load_all_devices() + self._load_all_devices() - if self._net_only: - LOGGER.info( - "Network only option configured, no tests will be run") - self._start_network() - else: - self._start_network() - self._test_orc.start() - self._net_orc.listener.register_callback( + if self._net_only: + LOGGER.info("Network only option configured, no tests will be run") + self._start_network() + else: + self._start_network() + self._test_orc.start() + + self._net_orc.listener.register_callback( self._device_stable, [NetworkEvent.DEVICE_STABLE] ) - LOGGER.info("Waiting for devices on the network...") + LOGGER.info("Waiting for devices on the network...") - # Check timeout and whether testing is currently in progress before stopping - time.sleep(RUNTIME) + # Check timeout and whether testing is currently in progress before stopping + time.sleep(RUNTIME) - self.stop() + self.stop() - def stop(self, kill=False): - self._stop_tests() - self._stop_network(kill=kill) + def stop(self, kill=False): + self._stop_tests() + self._stop_network(kill=kill) - def _register_exits(self): - signal.signal(signal.SIGINT, self._exit_handler) - signal.signal(signal.SIGTERM, self._exit_handler) - signal.signal(signal.SIGABRT, self._exit_handler) - signal.signal(signal.SIGQUIT, self._exit_handler) + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) - def _exit_handler(self, signum, arg): # pylint: disable=unused-argument - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received.") - self.stop(kill=True) - sys.exit(1) + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received.") + self.stop(kill=True) + sys.exit(1) - def _get_config_abs(self, config_file=None): - if config_file is None: - # If not defined, use relative pathing to local file - config_file = os.path.join(parent_dir, CONFIG_FILE) + def _get_config_abs(self, config_file=None): + if config_file is None: + # If not defined, use relative pathing to local file + config_file = os.path.join(parent_dir, CONFIG_FILE) - # Expand the config file to absolute pathing - return os.path.abspath(config_file) + # Expand the config file to absolute pathing + return os.path.abspath(config_file) - def _start_network(self): - # Load in local device configs to the network orchestrator - self._net_orc._devices = self._devices + def _start_network(self): + # Load in local device configs to the network orchestrator + self._net_orc._devices = self._devices - # Start the network orchestrator - self._net_orc.start() + # Start the network orchestrator + self._net_orc.start() - def _run_tests(self, device): - """Iterate through and start all test modules.""" + def _run_tests(self, device): + """Iterate through and start all test modules.""" - # TODO: Make this configurable - time.sleep(60) # Let device bootup + # To Do: Make this configurable + time.sleep(60) # Let device bootup - self._test_orc.run_test_modules(device) + self._test_orc._run_test_modules(device) - def _stop_network(self, kill=False): - self._net_orc.stop(kill=kill) + def _stop_network(self, kill=False): + self._net_orc.stop(kill=kill) - def _stop_tests(self): - self._test_orc.stop() + def _stop_tests(self): + self._test_orc.stop() - def _load_all_devices(self): - self._load_devices(device_dir=LOCAL_DEVICES_DIR) - LOGGER.info('Loaded ' + str(len(self._devices)) + ' devices') + def _load_all_devices(self): + self._load_devices(device_dir=LOCAL_DEVICES_DIR) + self._load_devices(device_dir=RESOURCE_DEVICES_DIR) - def _load_devices(self, device_dir): - LOGGER.debug('Loading devices from ' + device_dir) + def _load_devices(self, device_dir): + LOGGER.debug('Loading devices from ' + device_dir) - os.makedirs(device_dir, exist_ok=True) + os.makedirs(device_dir, exist_ok=True) - for device_folder in os.listdir(device_dir): - with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), - encoding='utf-8') as device_config_file: - device_config_json = json.load(device_config_file) + for device_folder in os.listdir(device_dir): + with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), + encoding='utf-8') as device_config_file: + device_config_json = json.load(device_config_file) - device_make = device_config_json.get(DEVICE_MAKE) - device_model = device_config_json.get(DEVICE_MODEL) - mac_addr = device_config_json.get(DEVICE_MAC_ADDR) - test_modules = device_config_json.get(DEVICE_TEST_MODULES) + device_make = device_config_json.get(DEVICE_MAKE) + device_model = device_config_json.get(DEVICE_MODEL) + mac_addr = device_config_json.get(DEVICE_MAC_ADDR) + test_modules = device_config_json.get(DEVICE_TEST_MODULES) - device = Device(make=device_make, model=device_model, - mac_addr=mac_addr, test_modules=json.dumps(test_modules)) - self._devices.append(device) - - def get_device(self, mac_addr): - """Returns a loaded device object from the device mac address.""" - for device in self._devices: - if device.mac_addr == mac_addr: - return device - return None - - def _device_discovered(self, mac_addr): - device = self.get_device(mac_addr) - if device is not None: - LOGGER.info( - f'Discovered {device.make} {device.model} on the network') - else: - device = Device(mac_addr=mac_addr) + device = Device(make=device_make, model=device_model, + mac_addr=mac_addr, test_modules=json.dumps(test_modules)) self._devices.append(device) - LOGGER.info( - f'A new device has been discovered with mac address {mac_addr}') - def _device_stable(self, mac_addr): - device = self.get_device(mac_addr) - LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') - self._test_orc.run_test_modules(device) + def get_device(self, mac_addr): + """Returns a loaded device object from the device mac address.""" + for device in self._devices: + if device.mac_addr == mac_addr: + return device + return None + + def _device_discovered(self, mac_addr): + device = self.get_device(mac_addr) + if device is not None: + LOGGER.info( + f'Discovered {device.make} {device.model} on the network') + else: + device = Device(mac_addr=mac_addr) + self._devices.append(device) + LOGGER.info( + f'A new device has been discovered with mac address {mac_addr}') + + def _device_stable(self, mac_addr): + device = self.get_device(mac_addr) + LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') + self._test_orc.run_test_modules(device) diff --git a/net_orc/python/src/network_orchestrator.py b/net_orc/python/src/network_orchestrator.py index 6930f22be..2950f97fb 100644 --- a/net_orc/python/src/network_orchestrator.py +++ b/net_orc/python/src/network_orchestrator.py @@ -1,620 +1,754 @@ -#!/usr/bin/env python3 - -import getpass -import ipaddress -import json -import os -import subprocess -import sys -import time -import threading - -import docker -from docker.types import Mount - -import logger -import util -from listener import Listener -from network_validator import NetworkValidator - -LOGGER = logger.get_logger("net_orc") -CONFIG_FILE = "conf/system.json" -EXAMPLE_CONFIG_FILE = "conf/system.json.example" -RUNTIME_DIR = "runtime/network" -NETWORK_MODULES_DIR = "network/modules" -NETWORK_MODULE_METADATA = "conf/module_config.json" -DEVICE_BRIDGE = "tr-d" -INTERNET_BRIDGE = "tr-c" -PRIVATE_DOCKER_NET = "tr-private-net" -CONTAINER_NAME = "network_orchestrator" -RUNTIME = 300 - - -class NetworkOrchestrator: - """Manage and controls a virtual testing network.""" - - def __init__(self, config_file=CONFIG_FILE, validate=True, async_monitor=False, single_intf = False): - self._int_intf = None - self._dev_intf = None - self._single_intf = single_intf - self.listener = None - - self._net_modules = [] - - self.validate = validate - - self.async_monitor = async_monitor - - self._path = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.realpath(__file__)))) - - self.validator = NetworkValidator() - - self.network_config = NetworkConfig() - - self.load_config(config_file) - - def start(self): - """Start the network orchestrator.""" - - LOGGER.info("Starting Network Orchestrator") - # Get all components ready - self.load_network_modules() - - # Restore the network first if required - self.stop(kill=True) - - self.start_network() - - if self.async_monitor: - # Run the monitor method asynchronously to keep this method non-blocking - self._monitor_thread = threading.Thread( - target=self.monitor_network) - self._monitor_thread.daemon = True - self._monitor_thread.start() - else: - self.monitor_network() - - def start_network(self): - """Start the virtual testing network.""" - LOGGER.info("Starting network") - - self.build_network_modules() - self.create_net() - self.start_network_services() - - if self.validate: - # Start the validator after network is ready - self.validator.start() - - # Get network ready (via Network orchestrator) - LOGGER.info("Network is ready.") - - def stop(self, kill=False): - """Stop the network orchestrator.""" - self.stop_validator(kill=kill) - self.stop_network(kill=kill) - - def stop_validator(self, kill=False): - """Stop the network validator.""" - # Shutdown the validator - self.validator.stop(kill=kill) - - def stop_network(self, kill=False): - """Stop the virtual testing network.""" - # Shutdown network - self.stop_networking_services(kill=kill) - self.restore_net() - - def monitor_network(self): - # TODO: This time should be configurable (How long to hold before exiting, this could be infinite too) - time.sleep(RUNTIME) - - self.stop() - - def load_config(self,config_file=None): - if config_file is None: - # If not defined, use relative pathing to local file - self._config_file=os.path.join(self._path, CONFIG_FILE) - else: - # If defined, use as provided - self._config_file=config_file - - if not os.path.isfile(self._config_file): - LOGGER.error("Configuration file is not present at " + config_file) - LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) - sys.exit(1) - - LOGGER.info("Loading config file: " + os.path.abspath(self._config_file)) - with open(self._config_file, encoding='UTF-8') as config_json_file: - config_json = json.load(config_json_file) - self.import_config(config_json) - - def import_config(self, json_config): - self._int_intf = json_config['network']['internet_intf'] - self._dev_intf = json_config['network']['device_intf'] - - def _check_network_services(self): - LOGGER.debug("Checking network modules...") - for net_module in self._net_modules: - if net_module.enable_container: - LOGGER.debug("Checking network module: " + - net_module.display_name) - success = self._ping(net_module) - if success: - LOGGER.debug(net_module.display_name + - " responded succesfully: " + str(success)) - else: - LOGGER.error(net_module.display_name + - " failed to respond to ping") - - def _ping(self, net_module): - host = net_module.net_config.ipv4_address - namespace = "tr-ctns-" + net_module.dir_name - cmd = "ip netns exec " + namespace + " ping -c 1 " + str(host) - success = util.run_command(cmd, output=False) - return success - - def _create_private_net(self): - client = docker.from_env() - try: - network = client.networks.get(PRIVATE_DOCKER_NET) - network.remove() - except docker.errors.NotFound: - pass - - # TODO: These should be made into variables - ipam_pool = docker.types.IPAMPool( - subnet='100.100.0.0/16', - iprange='100.100.100.0/24' - ) - - ipam_config = docker.types.IPAMConfig( - pool_configs=[ipam_pool] - ) - - client.networks.create( - PRIVATE_DOCKER_NET, - ipam=ipam_config, - internal=True, - check_duplicate=True, - driver="macvlan" - ) - - def _ci_pre_network_create(self): - """ Stores network properties to restore network after - network creation and flushes internet interface - """ - - self._ethmac = subprocess.check_output( - f"cat /sys/class/net/{self._int_intf}/address", shell=True).decode("utf-8").strip() - self._gateway = subprocess.check_output( - "ip route | head -n 1 | awk '{print $3}'", shell=True).decode("utf-8").strip() - self._ipv4 = subprocess.check_output( - f"ip a show {self._int_intf} | grep \"inet \" | awk '{{print $2}}'", shell=True).decode("utf-8").strip() - self._ipv6 = subprocess.check_output( - f"ip a show {self._int_intf} | grep inet6 | awk '{{print $2}}'", shell=True).decode("utf-8").strip() - self._brd = subprocess.check_output( - f"ip a show {self._int_intf} | grep \"inet \" | awk '{{print $4}}'", shell=True).decode("utf-8").strip() - - def _ci_post_network_create(self): - """ Restore network connection in CI environment """ - LOGGER.info("post cr") - util.run_command(f"ip address del {self._ipv4} dev {self._int_intf}") - util.run_command(f"ip -6 address del {self._ipv6} dev {self._int_intf}") - util.run_command(f"ip link set dev {self._int_intf} address 00:B0:D0:63:C2:26") - util.run_command(f"ip addr flush dev {self._int_intf}") - util.run_command(f"ip addr add dev {self._int_intf} 0.0.0.0") - util.run_command(f"ip addr add dev {INTERNET_BRIDGE} {self._ipv4} broadcast {self._brd}") - util.run_command(f"ip -6 addr add {self._ipv6} dev {INTERNET_BRIDGE} ") - util.run_command(f"systemd-resolve --interface {INTERNET_BRIDGE} --set-dns 8.8.8.8") - util.run_command(f"ip link set dev {INTERNET_BRIDGE} up") - util.run_command(f"dhclient {INTERNET_BRIDGE}") - util.run_command(f"ip route del default via 10.1.0.1") - util.run_command(f"ip route add default via {self._gateway} src {self._ipv4[:-3]} metric 100 dev {INTERNET_BRIDGE}") - - def create_net(self): - LOGGER.info("Creating baseline network") - - if not util.interface_exists(self._int_intf) or not util.interface_exists(self._dev_intf): - LOGGER.error("Configured interfaces are not ready for use. " + - "Ensure both interfaces are connected.") - sys.exit(1) - - if self._single_intf: - self._ci_pre_network_create() - - # Create data plane - util.run_command("ovs-vsctl add-br " + DEVICE_BRIDGE) - - # Create control plane - util.run_command("ovs-vsctl add-br " + INTERNET_BRIDGE) - - # Add external interfaces to data and control plane - util.run_command("ovs-vsctl add-port " + - DEVICE_BRIDGE + " " + self._dev_intf) - util.run_command("ovs-vsctl add-port " + - INTERNET_BRIDGE + " " + self._int_intf) - - # Enable forwarding of eapol packets - util.run_command("ovs-ofctl add-flow " + DEVICE_BRIDGE + - " 'table=0, dl_dst=01:80:c2:00:00:03, actions=flood'") - - # Remove IP from internet adapter - util.run_command("ifconfig " + self._int_intf + " 0.0.0.0") - - # Set ports up - util.run_command("ip link set dev " + DEVICE_BRIDGE + " up") - util.run_command("ip link set dev " + INTERNET_BRIDGE + " up") - - if self._single_intf: - self._ci_post_network_create() - - self._create_private_net() - - self.listener = Listener(self._dev_intf) - self.listener.start_listener() - - def load_network_modules(self): - """Load network modules from module_config.json.""" - LOGGER.debug("Loading network modules from /" + NETWORK_MODULES_DIR) - - loaded_modules = "Loaded the following network modules: " - net_modules_dir = os.path.join(self._path, NETWORK_MODULES_DIR) - - for module_dir in os.listdir(net_modules_dir): - - if self._get_network_module(module_dir) is None: - loaded_module = self._load_network_module(module_dir) - loaded_modules += loaded_module.dir_name + " " - - LOGGER.info(loaded_modules) - - def _load_network_module(self, module_dir): - - net_modules_dir = os.path.join(self._path, NETWORK_MODULES_DIR) - - net_module = NetworkModule() - - # Load basic module information - net_module_json = json.load(open(os.path.join( - self._path, net_modules_dir, module_dir, NETWORK_MODULE_METADATA), encoding='UTF-8')) - - net_module.name = net_module_json['config']['meta']['name'] - net_module.display_name = net_module_json['config']['meta']['display_name'] - net_module.description = net_module_json['config']['meta']['description'] - net_module.dir = os.path.join( - self._path, net_modules_dir, module_dir) - net_module.dir_name = module_dir - net_module.build_file = module_dir + ".Dockerfile" - net_module.container_name = "tr-ct-" + net_module.dir_name - net_module.image_name = "test-run/" + net_module.dir_name - - # Attach folder mounts to network module - if "docker" in net_module_json['config']: - - if "mounts" in net_module_json['config']['docker']: - for mount_point in net_module_json['config']['docker']['mounts']: - net_module.mounts.append(Mount( - target=mount_point['target'], - source=os.path.join( - os.getcwd(), mount_point['source']), - type='bind' - )) - - if "depends_on" in net_module_json['config']['docker']: - depends_on_module = net_module_json['config']['docker']['depends_on'] - if self._get_network_module(depends_on_module) is None: - self._load_network_module(depends_on_module) - - # Determine if this is a container or just an image/template - if "enable_container" in net_module_json['config']['docker']: - net_module.enable_container = net_module_json['config']['docker']['enable_container'] - - # Load network service networking configuration - if net_module.enable_container: - - net_module.net_config.enable_wan = net_module_json['config']['network']['enable_wan'] - net_module.net_config.ip_index = net_module_json['config']['network']['ip_index'] - - net_module.net_config.host = False if not "host" in net_module_json[ - 'config']['network'] else net_module_json['config']['network']['host'] - - net_module.net_config.ipv4_address = self.network_config.ipv4_network[ - net_module.net_config.ip_index] - net_module.net_config.ipv4_network = self.network_config.ipv4_network - - net_module.net_config.ipv6_address = self.network_config.ipv6_network[ - net_module.net_config.ip_index] - net_module.net_config.ipv6_network = self.network_config.ipv6_network - - self._net_modules.append(net_module) - return net_module - - def build_network_modules(self): - LOGGER.info("Building network modules...") - for net_module in self._net_modules: - self._build_module(net_module) - - def _build_module(self, net_module): - LOGGER.debug("Building network module " + net_module.dir_name) - client = docker.from_env() - client.images.build( - dockerfile=os.path.join(net_module.dir, net_module.build_file), - path=self._path, - forcerm=True, - tag="test-run/" + net_module.dir_name - ) - - def _get_network_module(self, name): - for net_module in self._net_modules: - if name == net_module.display_name or name == net_module.name or name == net_module.dir_name: - return net_module - return None - - # Start the OVS network module - # This should always be called before loading all - # other modules to allow for a properly setup base - # network - def _start_ovs_module(self): - self._start_network_service(self._get_network_module("OVS")) - - def _start_network_service(self, net_module): - - LOGGER.debug("Starting net service " + net_module.display_name) - network = "host" if net_module.net_config.host else PRIVATE_DOCKER_NET - LOGGER.debug(f"""Network: {network}, image name: {net_module.image_name}, - container name: {net_module.container_name}""") - try: - client = docker.from_env() - net_module.container = client.containers.run( - net_module.image_name, - auto_remove=True, - cap_add=["NET_ADMIN"], - name=net_module.container_name, - hostname=net_module.container_name, - network=PRIVATE_DOCKER_NET, - privileged=True, - detach=True, - mounts=net_module.mounts, - environment={"HOST_USER": getpass.getuser()} - ) - except docker.errors.ContainerError as error: - LOGGER.error("Container run error") - LOGGER.error(error) - - if network != "host": - self._attach_service_to_network(net_module) - - def _stop_service_module(self, net_module, kill=False): - LOGGER.debug("Stopping Service container " + net_module.container_name) - try: - container = self._get_service_container(net_module) - if container is not None: - if kill: - LOGGER.debug("Killing container:" + - net_module.container_name) - container.kill() - else: - LOGGER.debug("Stopping container:" + - net_module.container_name) - container.stop() - LOGGER.debug("Container stopped:" + net_module.container_name) - except Exception as error: - LOGGER.error("Container stop error") - LOGGER.error(error) - - def _get_service_container(self, net_module): - LOGGER.debug("Resolving service container: " + - net_module.container_name) - container = None - try: - client = docker.from_env() - container = client.containers.get(net_module.container_name) - except docker.errors.NotFound: - LOGGER.debug("Container " + - net_module.container_name + " not found") - except Exception as e: - LOGGER.error("Failed to resolve container") - LOGGER.error(e) - return container - - def stop_networking_services(self, kill=False): - LOGGER.info("Stopping network services") - for net_module in self._net_modules: - # Network modules may just be Docker images, so we do not want to stop them - if not net_module.enable_container: - continue - self._stop_service_module(net_module, kill) - - def start_network_services(self): - LOGGER.info("Starting network services") - - os.makedirs(os.path.join(os.getcwd(), RUNTIME_DIR), exist_ok=True) - - for net_module in self._net_modules: - - # TODO: There should be a better way of doing this - # Do not try starting OVS module again, as it should already be running - if "OVS" != net_module.display_name: - - # Network modules may just be Docker images, so we do not want to start them as containers - if not net_module.enable_container: - continue - - self._start_network_service(net_module) - - LOGGER.info("All network services are running") - self._check_network_services() - - # TODO: Let's move this into a separate script? It does not look great - def _attach_service_to_network(self, net_module): - LOGGER.debug("Attaching net service " + - net_module.display_name + " to device bridge") - - # Device bridge interface example: tr-di-dhcp (Test Run Device Interface for DHCP container) - bridge_intf = DEVICE_BRIDGE + "i-" + net_module.dir_name - - # Container interface example: tr-cti-dhcp (Test Run Container Interface for DHCP container) - container_intf = "tr-cti-" + net_module.dir_name - - # Container network namespace name - container_net_ns = "tr-ctns-" + net_module.dir_name - - # Create interface pair - util.run_command("ip link add " + bridge_intf + - " type veth peer name " + container_intf) - - # Add bridge interface to device bridge - util.run_command("ovs-vsctl add-port " + - DEVICE_BRIDGE + " " + bridge_intf) - - # Get PID for running container - # TODO: Some error checking around missing PIDs might be required - container_pid = util.run_command( - "docker inspect -f {{.State.Pid}} " + net_module.container_name)[0] - - # Create symlink for container network namespace - util.run_command("ln -sf /proc/" + container_pid + - "/ns/net /var/run/netns/" + container_net_ns) - - # Attach container interface to container network namespace - util.run_command("ip link set " + container_intf + - " netns " + container_net_ns) - - # Rename container interface name to veth0 - util.run_command("ip netns exec " + container_net_ns + - " ip link set dev " + container_intf + " name veth0") - - # Set MAC address of container interface - util.run_command("ip netns exec " + container_net_ns + " ip link set dev veth0 address 9a:02:57:1e:8f:" + str(net_module.net_config.ip_index)) - - # Set IP address of container interface - util.run_command("ip netns exec " + container_net_ns + " ip addr add " + - net_module.net_config.get_ipv4_addr_with_prefix() + " dev veth0") - - util.run_command("ip netns exec " + container_net_ns + " ip addr add " + - net_module.net_config.get_ipv6_addr_with_prefix() + " dev veth0") - - # Set interfaces up - util.run_command("ip link set dev " + bridge_intf + " up") - util.run_command("ip netns exec " + container_net_ns + - " ip link set dev veth0 up") - - if net_module.net_config.enable_wan: - LOGGER.debug("Attaching net service " + - net_module.display_name + " to internet bridge") - - # Internet bridge interface example: tr-ci-dhcp (Test Run Control (Internet) Interface for DHCP container) - bridge_intf = INTERNET_BRIDGE + "i-" + net_module.dir_name - - # Container interface example: tr-cti-dhcp (Test Run Container Interface for DHCP container) - container_intf = "tr-cti-" + net_module.dir_name - - # Create interface pair - util.run_command("ip link add " + bridge_intf + - " type veth peer name " + container_intf) - - # Attach bridge interface to internet bridge - util.run_command("ovs-vsctl add-port " + - INTERNET_BRIDGE + " " + bridge_intf) - - # Attach container interface to container network namespace - util.run_command("ip link set " + container_intf + - " netns " + container_net_ns) - - # Rename container interface name to eth1 - util.run_command("ip netns exec " + container_net_ns + - " ip link set dev " + container_intf + " name eth1") - - # Set MAC address of container interface - util.run_command("ip netns exec " + container_net_ns + - " ip link set dev eth1 address 9a:02:57:1e:8f:0" + str(net_module.net_config.ip_index)) - - # Set interfaces up - util.run_command("ip link set dev " + bridge_intf + " up") - util.run_command("ip netns exec " + - container_net_ns + " ip link set dev eth1 up") - - def restore_net(self): - - LOGGER.info("Clearing baseline network") - - if hasattr(self, 'listener') and self.listener is not None and self.listener.is_running(): - self.listener.stop_listener() - - client = docker.from_env() - - # Stop all network containers if still running - for net_module in self._net_modules: - try: - container = client.containers.get( - "tr-ct-" + net_module.dir_name) - container.kill() - except Exception: - continue - - # Delete data plane - util.run_command("ovs-vsctl --if-exists del-br tr-d") - - # Delete control plane - util.run_command("ovs-vsctl --if-exists del-br tr-c") - - # Restart internet interface - if util.interface_exists(self._int_intf): - util.run_command("ip link set " + self._int_intf + " down") - util.run_command("ip link set " + self._int_intf + " up") - - LOGGER.info("Network is restored") - -class NetworkModule: - - def __init__(self): - self.name = None - self.display_name = None - self.description = None - - self.container = None - self.container_name = None - self.image_name = None - - # Absolute path - self.dir = None - self.dir_name = None - self.build_file = None - self.mounts = [] - - self.enable_container = True - - self.net_config = NetworkModuleNetConfig() - -# The networking configuration for a network module - -class NetworkModuleNetConfig: - - def __init__(self): - - self.enable_wan = False - - self.ip_index = 0 - self.ipv4_address = None - self.ipv4_network = None - self.ipv6_address = None - self.ipv6_network = None - - self.host = False - - def get_ipv4_addr_with_prefix(self): - return format(self.ipv4_address) + "/" + str(self.ipv4_network.prefixlen) - - def get_ipv6_addr_with_prefix(self): - return format(self.ipv6_address) + "/" + str(self.ipv6_network.prefixlen) - -# Represents the current configuration of the network for the device bridge - -class NetworkConfig: - - # TODO: Let's get this from a configuration file - def __init__(self): - self.ipv4_network = ipaddress.ip_network('10.10.10.0/24') - self.ipv6_network = ipaddress.ip_network('fd10:77be:4186::/64') +#!/usr/bin/env python3 + +import binascii +import getpass +import ipaddress +import json +import os +from scapy.all import BOOTP +import shutil +import subprocess +import sys +import time +import threading +from threading import Timer +import docker +from docker.types import Mount +import logger +import util +from listener import Listener +from network_device import NetworkDevice +from network_event import NetworkEvent +from network_validator import NetworkValidator + +LOGGER = logger.get_logger("net_orc") +CONFIG_FILE = "conf/system.json" +EXAMPLE_CONFIG_FILE = "conf/system.json.example" +RUNTIME_DIR = "runtime/network" +NETWORK_MODULES_DIR = "network/modules" +NETWORK_MODULE_METADATA = "conf/module_config.json" +DEVICE_BRIDGE = "tr-d" +INTERNET_BRIDGE = "tr-c" +PRIVATE_DOCKER_NET = "tr-private-net" +CONTAINER_NAME = "network_orchestrator" + +RUNTIME_KEY = "runtime" +MONITOR_PERIOD_KEY = "monitor_period" +STARTUP_TIMEOUT_KEY = "startup_timeout" +DEFAULT_STARTUP_TIMEOUT = 60 +DEFAULT_RUNTIME = 1200 +DEFAULT_MONITOR_PERIOD = 300 + +RUNTIME = 1500 + + +class NetworkOrchestrator: + """Manage and controls a virtual testing network.""" + + def __init__(self, config_file=CONFIG_FILE, validate=True, async_monitor=False, single_intf = False): + + self._runtime = DEFAULT_RUNTIME + self._startup_timeout = DEFAULT_STARTUP_TIMEOUT + self._monitor_period = DEFAULT_MONITOR_PERIOD + + self._int_intf = None + self._dev_intf = None + self._single_intf = single_intf + + self.listener = None + + self._net_modules = [] + + self.validate = validate + + self.async_monitor = async_monitor + + self._path = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__)))) + + self.validator = NetworkValidator() + + shutil.rmtree(os.path.join(os.getcwd(), RUNTIME_DIR), ignore_errors=True) + + self.network_config = NetworkConfig() + + self.load_config(config_file) + + def start(self): + """Start the network orchestrator.""" + + LOGGER.info("Starting Network Orchestrator") + # Get all components ready + self.load_network_modules() + + # Restore the network first if required + self.stop(kill=True) + + self.start_network() + + if self.async_monitor: + # Run the monitor method asynchronously to keep this method non-blocking + self._monitor_thread = threading.Thread( + target=self.monitor_network) + self._monitor_thread.daemon = True + self._monitor_thread.start() + else: + self.monitor_network() + + def start_network(self): + """Start the virtual testing network.""" + LOGGER.info("Starting network") + + self.build_network_modules() + self.create_net() + self.start_network_services() + + if self.validate: + # Start the validator after network is ready + self.validator.start() + + # Get network ready (via Network orchestrator) + LOGGER.info("Network is ready.") + + def stop(self, kill=False): + """Stop the network orchestrator.""" + self.stop_validator(kill=kill) + self.stop_network(kill=kill) + + def stop_validator(self, kill=False): + """Stop the network validator.""" + # Shutdown the validator + self.validator.stop(kill=kill) + + def stop_network(self, kill=False): + """Stop the virtual testing network.""" + # Shutdown network + self.stop_networking_services(kill=kill) + self.restore_net() + + def monitor_network(self): + # TODO: This time should be configurable (How long to hold before exiting, this could be infinite too) + time.sleep(RUNTIME) + + self.stop() + + def load_config(self,config_file=None): + if config_file is None: + # If not defined, use relative pathing to local file + self._config_file=os.path.join(self._path, CONFIG_FILE) + else: + # If defined, use as provided + self._config_file=config_file + + if not os.path.isfile(self._config_file): + LOGGER.error("Configuration file is not present at " + config_file) + LOGGER.info("An example is present in " + EXAMPLE_CONFIG_FILE) + sys.exit(1) + + LOGGER.info("Loading config file: " + os.path.abspath(self._config_file)) + with open(self._config_file, encoding='UTF-8') as config_json_file: + config_json = json.load(config_json_file) + self.import_config(config_json) + + def _device_discovered(self, mac_addr): + + LOGGER.debug(f'Discovered device {mac_addr}. Waiting for device to obtain IP') + device = self._get_device(mac_addr=mac_addr) + + timeout = time.time() + self._startup_timeout + + while time.time() < timeout: + if device.ip_addr is None: + time.sleep(3) + else: + break + + if device.ip_addr is None: + LOGGER.info(f"Timed out whilst waiting for {mac_addr} to obtain an IP address") + return + + LOGGER.info(f"Device with mac addr {device.mac_addr} has obtained IP address {device.ip_addr}") + + self._start_device_monitor(device) + + def _dhcp_lease_ack(self, packet): + mac_addr = packet[BOOTP].chaddr.hex(":")[0:17] + device = self._get_device(mac_addr=mac_addr) + device.ip_addr = packet[BOOTP].yiaddr + + def _start_device_monitor(self, device): + """Start a timer until the steady state has been reached and + callback the steady state method for this device.""" + LOGGER.info(f"Monitoring device with mac addr {device.mac_addr} for {str(self._monitor_period)} seconds") + timer = Timer(self._monitor_period, + self.listener.call_callback, + args=(NetworkEvent.DEVICE_STABLE, device.mac_addr,)) + timer.start() + + def _get_device(self, mac_addr): + for device in self._devices: + if device.mac_addr == mac_addr: + return device + device = NetworkDevice(mac_addr=mac_addr) + self._devices.append(device) + return device + + def import_config(self, json_config): + self._int_intf = json_config['network']['internet_intf'] + self._dev_intf = json_config['network']['device_intf'] + + if RUNTIME_KEY in json_config: + self._runtime = json_config[RUNTIME_KEY] + if STARTUP_TIMEOUT_KEY in json_config: + self._startup_timeout = json_config[STARTUP_TIMEOUT_KEY] + if MONITOR_PERIOD_KEY in json_config: + self._monitor_period = json_config[MONITOR_PERIOD_KEY] + + def _check_network_services(self): + LOGGER.debug("Checking network modules...") + for net_module in self._net_modules: + if net_module.enable_container: + LOGGER.debug("Checking network module: " + + net_module.display_name) + success = self._ping(net_module) + if success: + LOGGER.debug(net_module.display_name + + " responded succesfully: " + str(success)) + else: + LOGGER.error(net_module.display_name + + " failed to respond to ping") + + def _ping(self, net_module): + host = net_module.net_config.ipv4_address + namespace = "tr-ctns-" + net_module.dir_name + cmd = "ip netns exec " + namespace + " ping -c 1 " + str(host) + success = util.run_command(cmd, output=False) + return success + + def _create_private_net(self): + client = docker.from_env() + try: + network = client.networks.get(PRIVATE_DOCKER_NET) + network.remove() + except docker.errors.NotFound: + pass + + # TODO: These should be made into variables + ipam_pool = docker.types.IPAMPool( + subnet='100.100.0.0/16', + iprange='100.100.100.0/24' + ) + + ipam_config = docker.types.IPAMConfig( + pool_configs=[ipam_pool] + ) + + client.networks.create( + PRIVATE_DOCKER_NET, + ipam=ipam_config, + internal=True, + check_duplicate=True, + driver="macvlan" + ) + + def _ci_pre_network_create(self): + """ Stores network properties to restore network after + network creation and flushes internet interface + """ + + self._ethmac = subprocess.check_output( + f"cat /sys/class/net/{self._int_intf}/address", shell=True).decode("utf-8").strip() + self._gateway = subprocess.check_output( + "ip route | head -n 1 | awk '{print $3}'", shell=True).decode("utf-8").strip() + self._ipv4 = subprocess.check_output( + f"ip a show {self._int_intf} | grep \"inet \" | awk '{{print $2}}'", shell=True).decode("utf-8").strip() + self._ipv6 = subprocess.check_output( + f"ip a show {self._int_intf} | grep inet6 | awk '{{print $2}}'", shell=True).decode("utf-8").strip() + self._brd = subprocess.check_output( + f"ip a show {self._int_intf} | grep \"inet \" | awk '{{print $4}}'", shell=True).decode("utf-8").strip() + + def _ci_post_network_create(self): + """ Restore network connection in CI environment """ + LOGGER.info("post cr") + util.run_command(f"ip address del {self._ipv4} dev {self._int_intf}") + util.run_command(f"ip -6 address del {self._ipv6} dev {self._int_intf}") + util.run_command(f"ip link set dev {self._int_intf} address 00:B0:D0:63:C2:26") + util.run_command(f"ip addr flush dev {self._int_intf}") + util.run_command(f"ip addr add dev {self._int_intf} 0.0.0.0") + util.run_command(f"ip addr add dev {INTERNET_BRIDGE} {self._ipv4} broadcast {self._brd}") + util.run_command(f"ip -6 addr add {self._ipv6} dev {INTERNET_BRIDGE} ") + util.run_command(f"systemd-resolve --interface {INTERNET_BRIDGE} --set-dns 8.8.8.8") + util.run_command(f"ip link set dev {INTERNET_BRIDGE} up") + util.run_command(f"dhclient {INTERNET_BRIDGE}") + util.run_command(f"ip route del default via 10.1.0.1") + util.run_command(f"ip route add default via {self._gateway} src {self._ipv4[:-3]} metric 100 dev {INTERNET_BRIDGE}") + + def create_net(self): + LOGGER.info("Creating baseline network") + + if not util.interface_exists(self._int_intf) or not util.interface_exists(self._dev_intf): + LOGGER.error("Configured interfaces are not ready for use. " + + "Ensure both interfaces are connected.") + sys.exit(1) + + if self._single_intf: + self._ci_pre_network_create() + + # Create data plane + util.run_command("ovs-vsctl add-br " + DEVICE_BRIDGE) + + # Create control plane + util.run_command("ovs-vsctl add-br " + INTERNET_BRIDGE) + + # Add external interfaces to data and control plane + util.run_command("ovs-vsctl add-port " + + DEVICE_BRIDGE + " " + self._dev_intf) + util.run_command("ovs-vsctl add-port " + + INTERNET_BRIDGE + " " + self._int_intf) + + # Enable forwarding of eapol packets + util.run_command("ovs-ofctl add-flow " + DEVICE_BRIDGE + + " 'table=0, dl_dst=01:80:c2:00:00:03, actions=flood'") + + # Remove IP from internet adapter + util.run_command("ifconfig " + self._int_intf + " 0.0.0.0") + + # Set ports up + util.run_command("ip link set dev " + DEVICE_BRIDGE + " up") + util.run_command("ip link set dev " + INTERNET_BRIDGE + " up") + + if self._single_intf: + self._ci_post_network_create() + + self._create_private_net() + + self.listener = Listener(self._dev_intf) + self.listener.register_callback(self._device_discovered, [ + NetworkEvent.DEVICE_DISCOVERED]) + self.listener.register_callback( + self._dhcp_lease_ack, [NetworkEvent.DHCP_LEASE_ACK]) + self.listener.start_listener() + + def load_network_modules(self): + """Load network modules from module_config.json.""" + LOGGER.debug("Loading network modules from /" + NETWORK_MODULES_DIR) + + loaded_modules = "Loaded the following network modules: " + net_modules_dir = os.path.join(self._path, NETWORK_MODULES_DIR) + + for module_dir in os.listdir(net_modules_dir): + + if self._get_network_module(module_dir) is None: + loaded_module = self._load_network_module(module_dir) + loaded_modules += loaded_module.dir_name + " " + + LOGGER.info(loaded_modules) + + def _load_network_module(self, module_dir): + + net_modules_dir = os.path.join(self._path, NETWORK_MODULES_DIR) + + net_module = NetworkModule() + + # Load basic module information + net_module_json = json.load(open(os.path.join( + self._path, net_modules_dir, module_dir, NETWORK_MODULE_METADATA), encoding='UTF-8')) + + net_module.name = net_module_json['config']['meta']['name'] + net_module.display_name = net_module_json['config']['meta']['display_name'] + net_module.description = net_module_json['config']['meta']['description'] + net_module.dir = os.path.join( + self._path, net_modules_dir, module_dir) + net_module.dir_name = module_dir + net_module.build_file = module_dir + ".Dockerfile" + net_module.container_name = "tr-ct-" + net_module.dir_name + net_module.image_name = "test-run/" + net_module.dir_name + + # Attach folder mounts to network module + if "docker" in net_module_json['config']: + + if "mounts" in net_module_json['config']['docker']: + for mount_point in net_module_json['config']['docker']['mounts']: + net_module.mounts.append(Mount( + target=mount_point['target'], + source=os.path.join( + os.getcwd(), mount_point['source']), + type='bind' + )) + + if "depends_on" in net_module_json['config']['docker']: + depends_on_module = net_module_json['config']['docker']['depends_on'] + if self._get_network_module(depends_on_module) is None: + self._load_network_module(depends_on_module) + + # Determine if this is a container or just an image/template + if "enable_container" in net_module_json['config']['docker']: + net_module.enable_container = net_module_json['config']['docker']['enable_container'] + + # Load network service networking configuration + if net_module.enable_container: + + net_module.net_config.enable_wan = net_module_json['config']['network']['enable_wan'] + net_module.net_config.ip_index = net_module_json['config']['network']['ip_index'] + + net_module.net_config.host = False if not "host" in net_module_json[ + 'config']['network'] else net_module_json['config']['network']['host'] + + net_module.net_config.ipv4_address = self.network_config.ipv4_network[ + net_module.net_config.ip_index] + net_module.net_config.ipv4_network = self.network_config.ipv4_network + + net_module.net_config.ipv6_address = self.network_config.ipv6_network[ + net_module.net_config.ip_index] + net_module.net_config.ipv6_network = self.network_config.ipv6_network + + self._net_modules.append(net_module) + return net_module + + def build_network_modules(self): + LOGGER.info("Building network modules...") + for net_module in self._net_modules: + self._build_module(net_module) + + def _build_module(self, net_module): + LOGGER.debug("Building network module " + net_module.dir_name) + client = docker.from_env() + client.images.build( + dockerfile=os.path.join(net_module.dir, net_module.build_file), + path=self._path, + forcerm=True, + tag="test-run/" + net_module.dir_name + ) + + def _get_network_module(self, name): + for net_module in self._net_modules: + if name == net_module.display_name or name == net_module.name or name == net_module.dir_name: + return net_module + return None + + # Start the OVS network module + # This should always be called before loading all + # other modules to allow for a properly setup base + # network + def _start_ovs_module(self): + self._start_network_service(self._get_network_module("OVS")) + + def _start_network_service(self, net_module): + + LOGGER.debug("Starting net service " + net_module.display_name) + network = "host" if net_module.net_config.host else PRIVATE_DOCKER_NET + LOGGER.debug(f"""Network: {network}, image name: {net_module.image_name}, + container name: {net_module.container_name}""") + try: + client = docker.from_env() + net_module.container = client.containers.run( + net_module.image_name, + auto_remove=True, + cap_add=["NET_ADMIN"], + name=net_module.container_name, + hostname=net_module.container_name, + network=PRIVATE_DOCKER_NET, + privileged=True, + detach=True, + mounts=net_module.mounts, + environment={"HOST_USER": getpass.getuser()} + ) + except docker.errors.ContainerError as error: + LOGGER.error("Container run error") + LOGGER.error(error) + + if network != "host": + self._attach_service_to_network(net_module) + + def _stop_service_module(self, net_module, kill=False): + LOGGER.debug("Stopping Service container " + net_module.container_name) + try: + container = self._get_service_container(net_module) + if container is not None: + if kill: + LOGGER.debug("Killing container:" + + net_module.container_name) + container.kill() + else: + LOGGER.debug("Stopping container:" + + net_module.container_name) + container.stop() + LOGGER.debug("Container stopped:" + net_module.container_name) + except Exception as error: + LOGGER.error("Container stop error") + LOGGER.error(error) + + def _get_service_container(self, net_module): + LOGGER.debug("Resolving service container: " + + net_module.container_name) + container = None + try: + client = docker.from_env() + container = client.containers.get(net_module.container_name) + except docker.errors.NotFound: + LOGGER.debug("Container " + + net_module.container_name + " not found") + except Exception as e: + LOGGER.error("Failed to resolve container") + LOGGER.error(e) + return container + + def stop_networking_services(self, kill=False): + LOGGER.info("Stopping network services") + for net_module in self._net_modules: + # Network modules may just be Docker images, so we do not want to stop them + if not net_module.enable_container: + continue + self._stop_service_module(net_module, kill) + + def start_network_services(self): + LOGGER.info("Starting network services") + + os.makedirs(os.path.join(os.getcwd(), RUNTIME_DIR), exist_ok=True) + + for net_module in self._net_modules: + + # TODO: There should be a better way of doing this + # Do not try starting OVS module again, as it should already be running + if "OVS" != net_module.display_name: + + # Network modules may just be Docker images, so we do not want to start them as containers + if not net_module.enable_container: + continue + + self._start_network_service(net_module) + + LOGGER.info("All network services are running") + self._check_network_services() + + def _attach_test_module_to_network(self, test_module): + LOGGER.debug("Attaching test module " + + test_module.display_name + " to device bridge") + + # Device bridge interface example: tr-di-baseline-test (Test Run Device Interface for baseline test container) + bridge_intf = DEVICE_BRIDGE + "i-" + test_module.dir_name + "-test" + + # Container interface example: tr-cti-baseline-test (Test Run Container Interface for baseline test container) + container_intf = "tr-test-" + test_module.dir_name + + # Container network namespace name + container_net_ns = "tr-test-" + test_module.dir_name + + # Create interface pair + util.run_command("ip link add " + bridge_intf + + " type veth peer name " + container_intf) + + # Add bridge interface to device bridge + util.run_command("ovs-vsctl add-port " + + DEVICE_BRIDGE + " " + bridge_intf) + + # Get PID for running container + # TODO: Some error checking around missing PIDs might be required + container_pid = util.run_command( + "docker inspect -f {{.State.Pid}} " + test_module.container_name)[0] + + # Create symlink for container network namespace + util.run_command("ln -sf /proc/" + container_pid + + "/ns/net /var/run/netns/" + container_net_ns) + + # Attach container interface to container network namespace + util.run_command("ip link set " + container_intf + + " netns " + container_net_ns) + + # Rename container interface name to veth0 + util.run_command("ip netns exec " + container_net_ns + + " ip link set dev " + container_intf + " name veth0") + + # Set MAC address of container interface + util.run_command("ip netns exec " + container_net_ns + " ip link set dev veth0 address 9a:02:57:1e:8f:" + str(test_module.ip_index)) + + # Set IP address of container interface + ipv4_address = self.network_config.ipv4_network[test_module.ip_index] + ipv6_address = self.network_config.ipv6_network[test_module.ip_index] + + ipv4_address_with_prefix=str(ipv4_address) + "/" + str(self.network_config.ipv4_network.prefixlen) + ipv6_address_with_prefix=str(ipv6_address) + "/" + str(self.network_config.ipv6_network.prefixlen) + + util.run_command("ip netns exec " + container_net_ns + " ip addr add " + + ipv4_address_with_prefix + " dev veth0") + + util.run_command("ip netns exec " + container_net_ns + " ip addr add " + + ipv6_address_with_prefix + " dev veth0") + + + # Set interfaces up + util.run_command("ip link set dev " + bridge_intf + " up") + util.run_command("ip netns exec " + container_net_ns + + " ip link set dev veth0 up") + + # TODO: Let's move this into a separate script? It does not look great + def _attach_service_to_network(self, net_module): + LOGGER.debug("Attaching net service " + + net_module.display_name + " to device bridge") + + # Device bridge interface example: tr-di-dhcp (Test Run Device Interface for DHCP container) + bridge_intf = DEVICE_BRIDGE + "i-" + net_module.dir_name + + # Container interface example: tr-cti-dhcp (Test Run Container Interface for DHCP container) + container_intf = "tr-cti-" + net_module.dir_name + + # Container network namespace name + container_net_ns = "tr-ctns-" + net_module.dir_name + + # Create interface pair + util.run_command("ip link add " + bridge_intf + + " type veth peer name " + container_intf) + + # Add bridge interface to device bridge + util.run_command("ovs-vsctl add-port " + + DEVICE_BRIDGE + " " + bridge_intf) + + # Get PID for running container + # TODO: Some error checking around missing PIDs might be required + container_pid = util.run_command( + "docker inspect -f {{.State.Pid}} " + net_module.container_name)[0] + + # Create symlink for container network namespace + util.run_command("ln -sf /proc/" + container_pid + + "/ns/net /var/run/netns/" + container_net_ns) + + # Attach container interface to container network namespace + util.run_command("ip link set " + container_intf + + " netns " + container_net_ns) + + # Rename container interface name to veth0 + util.run_command("ip netns exec " + container_net_ns + + " ip link set dev " + container_intf + " name veth0") + + # Set MAC address of container interface + util.run_command("ip netns exec " + container_net_ns + " ip link set dev veth0 address 9a:02:57:1e:8f:" + str(net_module.net_config.ip_index)) + + # Set IP address of container interface + util.run_command("ip netns exec " + container_net_ns + " ip addr add " + + net_module.net_config.get_ipv4_addr_with_prefix() + " dev veth0") + + util.run_command("ip netns exec " + container_net_ns + " ip addr add " + + net_module.net_config.get_ipv6_addr_with_prefix() + " dev veth0") + + # Set interfaces up + util.run_command("ip link set dev " + bridge_intf + " up") + util.run_command("ip netns exec " + container_net_ns + + " ip link set dev veth0 up") + + if net_module.net_config.enable_wan: + LOGGER.debug("Attaching net service " + + net_module.display_name + " to internet bridge") + + # Internet bridge interface example: tr-ci-dhcp (Test Run Control (Internet) Interface for DHCP container) + bridge_intf = INTERNET_BRIDGE + "i-" + net_module.dir_name + + # Container interface example: tr-cti-dhcp (Test Run Container Interface for DHCP container) + container_intf = "tr-cti-" + net_module.dir_name + + # Create interface pair + util.run_command("ip link add " + bridge_intf + + " type veth peer name " + container_intf) + + # Attach bridge interface to internet bridge + util.run_command("ovs-vsctl add-port " + + INTERNET_BRIDGE + " " + bridge_intf) + + # Attach container interface to container network namespace + util.run_command("ip link set " + container_intf + + " netns " + container_net_ns) + + # Rename container interface name to eth1 + util.run_command("ip netns exec " + container_net_ns + + " ip link set dev " + container_intf + " name eth1") + + # Set MAC address of container interface + util.run_command("ip netns exec " + container_net_ns + + " ip link set dev eth1 address 9a:02:57:1e:8f:0" + str(net_module.net_config.ip_index)) + + # Set interfaces up + util.run_command("ip link set dev " + bridge_intf + " up") + util.run_command("ip netns exec " + + container_net_ns + " ip link set dev eth1 up") + + def restore_net(self): + + LOGGER.info("Clearing baseline network") + + if hasattr(self, 'listener') and self.listener is not None and self.listener.is_running(): + self.listener.stop_listener() + + client = docker.from_env() + + # Stop all network containers if still running + for net_module in self._net_modules: + try: + container = client.containers.get( + "tr-ct-" + net_module.dir_name) + container.kill() + except Exception: + continue + + # Delete data plane + util.run_command("ovs-vsctl --if-exists del-br tr-d") + + # Delete control plane + util.run_command("ovs-vsctl --if-exists del-br tr-c") + + # Restart internet interface + if util.interface_exists(self._int_intf): + util.run_command("ip link set " + self._int_intf + " down") + util.run_command("ip link set " + self._int_intf + " up") + + LOGGER.info("Network is restored") + +class NetworkModule: + + def __init__(self): + self.name = None + self.display_name = None + self.description = None + + self.container = None + self.container_name = None + self.image_name = None + + # Absolute path + self.dir = None + self.dir_name = None + self.build_file = None + self.mounts = [] + + self.enable_container = True + + self.net_config = NetworkModuleNetConfig() + +# The networking configuration for a network module + +class NetworkModuleNetConfig: + + def __init__(self): + + self.enable_wan = False + + self.ip_index = 0 + self.ipv4_address = None + self.ipv4_network = None + self.ipv6_address = None + self.ipv6_network = None + + self.host = False + + def get_ipv4_addr_with_prefix(self): + return format(self.ipv4_address) + "/" + str(self.ipv4_network.prefixlen) + + def get_ipv6_addr_with_prefix(self): + return format(self.ipv6_address) + "/" + str(self.ipv6_network.prefixlen) + +# Represents the current configuration of the network for the device bridge + +class NetworkConfig: + + # TODO: Let's get this from a configuration file + def __init__(self): + self.ipv4_network = ipaddress.ip_network('10.10.10.0/24') + self.ipv6_network = ipaddress.ip_network('fd10:77be:4186::/64') diff --git a/resources/devices/Template/device_config.json b/resources/devices/Template/device_config.json index f8b56b7a3..7a3d4441c 100644 --- a/resources/devices/Template/device_config.json +++ b/resources/devices/Template/device_config.json @@ -27,6 +27,121 @@ "enabled": true } } + }, + "nmap": { + "enabled": true, + "tests": { + "security.nmap.ports": { + "enabled": true, + "security.services.ftp": { + "tcp_ports": { + "20": { + "allowed": false + }, + "21": { + "allowed": false + } + } + }, + "security.services.ssh": { + "tcp_ports": { + "22": { + "allowed": true + } + } + }, + "security.services.telnet": { + "tcp_ports": { + "23": { + "allowed": false + } + } + }, + "security.services.smtp": { + "tcp_ports": { + "25": { + "allowed": false + }, + "465": { + "allowed": false + }, + "587": { + "allowed": false + } + } + }, + "security.services.http": { + "tcp_ports": { + "80": { + "allowed": false + } + } + }, + "security.services.pop": { + "tcp_ports": { + "110": { + "allowed": false + } + } + }, + "security.services.imap": { + "tcp_ports": { + "143": { + "allowed": false + } + } + }, + "security.services.snmpv3": { + "tcp_ports": { + "161": { + "allowed": false + }, + "162": { + "allowed": false + } + }, + "udp_ports": { + "161": { + "allowed": false + }, + "162": { + "allowed": false + } + } + }, + "security.services.https": { + "tcp_ports": { + "80": { + "allowed": false + } + } + }, + "security.services.vnc": { + "tcp_ports": { + "5500": { + "allowed": false + }, + "5800": { + "allowed": false + } + } + }, + "security.services.tftp": { + "udp_ports": { + "69": { + "allowed": false + } + } + }, + "security.services.ntp": { + "udp_ports": { + "123": { + "allowed": false + } + } + } + } + } } } } \ No newline at end of file diff --git a/test_orc/modules/base/base.Dockerfile b/test_orc/modules/base/base.Dockerfile index b5f35326a..a508caef7 100644 --- a/test_orc/modules/base/base.Dockerfile +++ b/test_orc/modules/base/base.Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:jammy # Install common software -RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix +RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix nmap --fix-missing # Setup the base python requirements COPY modules/base/python /testrun/python diff --git a/test_orc/modules/base/bin/get_ipv4_addr b/test_orc/modules/base/bin/get_ipv4_addr new file mode 100644 index 000000000..09a19bc13 --- /dev/null +++ b/test_orc/modules/base/bin/get_ipv4_addr @@ -0,0 +1,8 @@ +#!/bin/bash + +NET=$1 +MAC=$2 + +IP_ADDR=$(nmap -sP $NET | grep -B 2 $MAC | head -n 1 | cut -d " " -f 5) + +echo $IP_ADDR \ No newline at end of file diff --git a/test_orc/modules/base/python/src/test_module.py b/test_orc/modules/base/python/src/test_module.py index 6f7f48c3a..9a348faa7 100644 --- a/test_orc/modules/base/python/src/test_module.py +++ b/test_orc/modules/base/python/src/test_module.py @@ -1,6 +1,7 @@ import json import logger import os +import util LOGGER = None RESULTS_DIR = "/runtime/output/" @@ -12,8 +13,12 @@ class TestModule: def __init__(self, module_name, log_name): self._module_name = module_name self._device_mac = os.environ['DEVICE_MAC'] + self._ipv4_subnet = os.environ['IPV4_SUBNET'] + self._ipv6_subnet = os.environ['IPV6_SUBNET'] self._add_logger(log_name=log_name, module_name=module_name) self._config = self._read_config() + self._device_ipv4_addr = None + self._device_ipv6_addr = None def _add_logger(self, log_name, module_name): global LOGGER @@ -34,8 +39,11 @@ def _get_device_tests(self, device_test_module): return [] else: for test in module_tests: + # Resolve device specific configurations for the test if it exists + # and update module test config with device config options if test["name"] in device_test_module["tests"]: - test["enabled"] = device_test_module["tests"][test["name"]]["enabled"] + dev_test_config = device_test_module["tests"][test["name"]] + test["config"].update(dev_test_config) return module_tests def _get_device_test_module(self): @@ -45,8 +53,10 @@ def _get_device_test_module(self): return None def run_tests(self): + if self._config["config"]["network"]: + self._device_ipv4_addr = self._get_device_ipv4() + LOGGER.info("Device IP Resolved: " + str(self._device_ipv4_addr)) tests = self._get_tests() - device_modules = os.environ['DEVICE_TEST_MODULES'] for test in tests: test_method_name = "_" + test["name"].replace(".", "_") result = None @@ -55,7 +65,11 @@ def run_tests(self): # Resolve the correct python method by test name and run test if hasattr(self, test_method_name): - result = getattr(self, test_method_name)() + if "config" in test: + result = getattr(self, test_method_name)( + config=test["config"]) + else: + result = getattr(self, test_method_name)() else: LOGGER.info("Test " + test["name"] + " not resolved. Skipping") @@ -82,3 +96,11 @@ def _write_results(self, results): f = open(results_file, "w", encoding="utf-8") f.write(results) f.close() + + def _get_device_ipv4(self): + command = '/testrun/bin/get_ipv4_addr {} {}'.format( + self._ipv4_subnet, self._device_mac.upper()) + text, err = util.run_command(command) + if text: + return text.split("\n")[0] + return None diff --git a/test_orc/modules/base/python/src/util.py b/test_orc/modules/base/python/src/util.py new file mode 100644 index 000000000..a2dcfbdb1 --- /dev/null +++ b/test_orc/modules/base/python/src/util.py @@ -0,0 +1,25 @@ +import subprocess +import shlex +import logger + +# Runs a process at the os level +# By default, returns the standard output and error output +# If the caller sets optional output parameter to False, +# will only return a boolean result indicating if it was +# succesful in running the command. Failure is indicated +# by any return code from the process other than zero. +def run_command(cmd, output=True): + success = False + LOGGER = logger.get_logger('util') + process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.returncode !=0 and output: + err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) + LOGGER.error("Command Failed: " + cmd) + LOGGER.error("Error: " + err_msg) + else: + success = True + if output: + return stdout.strip().decode('utf-8'), stderr + else: + return success diff --git a/test_orc/modules/nmap/bin/start_test_module b/test_orc/modules/nmap/bin/start_test_module new file mode 100644 index 000000000..4bb7e9f96 --- /dev/null +++ b/test_orc/modules/nmap/bin/start_test_module @@ -0,0 +1,42 @@ +#!/bin/bash + +# An example startup script that does the bare minimum to start +# a test module via a pyhon script. Each test module should include a +# start_test_module file that overwrites this one to boot all of its +# specific requirements to run. + +# Define where the python source files are located +PYTHON_SRC_DIR=/testrun/python/src + +# Fetch module name +MODULE_NAME=$1 + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Allow a user to define an interface by passing it into this script +DEFINED_IFACE=$2 + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No interface defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +# Create and set permissions on the log files +LOG_FILE=/runtime/output/$MODULE_NAME.log +RESULT_FILE=/runtime/output/$MODULE_NAME-result.json +touch $LOG_FILE +touch $RESULT_FILE +chown $HOST_USER:$HOST_USER $LOG_FILE +chown $HOST_USER:$HOST_USER $RESULT_FILE + +# Run the python scrip that will execute the tests for this module +# -u flag allows python print statements +# to be logged by docker by running unbuffered +python3 -u $PYTHON_SRC_DIR/run.py "-m $MODULE_NAME" + +echo Module has finished \ No newline at end of file diff --git a/test_orc/modules/nmap/conf/module_config.json b/test_orc/modules/nmap/conf/module_config.json new file mode 100644 index 000000000..5449327a1 --- /dev/null +++ b/test_orc/modules/nmap/conf/module_config.json @@ -0,0 +1,176 @@ +{ + "config": { + "meta": { + "name": "nmap", + "display_name": "nmap", + "description": "Scan for open ports using nmap" + }, + "network": true, + "docker": { + "enable_container": true, + "timeout": 600 + }, + "tests": [ + { + "name": "security.nmap.ports", + "description": "Run an nmap scan of open ports", + "expected_behavior": "Report all open ports", + "config": { + "security.services.ftp": { + "tcp_ports": { + "20": { + "allowed": false, + "description": "File Transfer Protocol (FTP) Server Data Transfer" + }, + "21": { + "allowed": false, + "description": "File Transfer Protocol (FTP) Server Data Transfer" + } + }, + "description": "Check FTP port 20/21 is disabled and FTP is not running on any port", + "expected_behavior": "There is no FTP service running on any port" + }, + "security.services.ssh": { + "tcp_ports": { + "22": { + "allowed": true, + "description": "Secure Shell (SSH) server" + } + }, + "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", + "expected_behavior": "There is no FTP service running on any port" + }, + "security.services.telnet": { + "tcp_ports": { + "23": { + "allowed": false, + "description": "Telnet Server" + } + }, + "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", + "expected_behavior": "There is no FTP service running on any port" + }, + "security.services.smtp": { + "tcp_ports": { + "25": { + "allowed": false, + "description": "Simple Mail Transfer Protocol (SMTP) Server" + }, + "465": { + "allowed": false, + "description": "Simple Mail Transfer Protocol over SSL (SMTPS) Server" + }, + "587": { + "allowed": false, + "description": "Simple Mail Transfer Protocol via TLS (SMTPS) Server" + } + }, + "description": "Check SMTP port 25 is disabled and ports 465 or 587 with SSL encryption are (not?) enabled and SMTP is not running on any port.", + "expected_behavior": "There is no smtp service running on any port" + }, + "security.services.http": { + "tcp_ports": { + "80": { + "service_scan": { + "script": "http-methods" + }, + "allowed": false, + "description": "Administrative Insecure Web-Server" + } + }, + "description": "Check that there is no HTTP server running on any port", + "expected_behavior": "Device is unreachable on port 80 (or any other port) and only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)" + }, + "security.services.pop": { + "tcp_ports": { + "110": { + "allowed": false, + "description": "Post Office Protocol v3 (POP3) Server" + } + }, + "description": "Check POP port 110 is disalbed and POP is not running on any port", + "expected_behavior": "There is no pop service running on any port" + }, + "security.services.imap": { + "tcp_ports": { + "143": { + "allowed": false, + "description": "Internet Message Access Protocol (IMAP) Server" + } + }, + "description": "Check IMAP port 143 is disabled and IMAP is not running on any port", + "expected_behavior": "There is no imap service running on any port" + }, + "security.services.snmpv3": { + "tcp_ports": { + "161": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP)" + }, + "162": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP) Trap" + } + }, + "udp_ports": { + "161": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP)" + }, + "162": { + "allowed": false, + "description": "Simple Network Management Protocol (SNMP) Trap" + } + }, + "description": "Check SNMP port 161/162 is disabled. If SNMP is an essential service, check it supports version 3", + "expected_behavior": "Device is unreachable on port 161 (or any other port) and device is unreachable on port 162 (or any other port) unless SNMP is essential in which case it is SNMPv3 is used." + }, + "security.services.https": { + "tcp_ports": { + "80": { + "allowed": false, + "description": "Administrative Secure Web-Server" + } + }, + "description": "Check that if there is a web server running it is running on a secure port.", + "expected_behavior": "Device only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)" + }, + "security.services.vnc": { + "tcp_ports": { + "5800": { + "allowed": false, + "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol Over HTTP" + }, + "5500": { + "allowed": false, + "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol" + } + }, + "description": "Check VNC is disabled on any port", + "expected_behavior": "Device cannot be accessed /connected to via VNc on any port" + }, + "security.services.tftp": { + "udp_ports": { + "69": { + "allowed": false, + "description": "Trivial File Transfer Protocol (TFTP) Server" + } + }, + "description": "Check TFTP port 69 is disabled (UDP)", + "expected_behavior": "There is no tftp service running on any port" + }, + "security.services.ntp": { + "udp_ports": { + "123": { + "allowed": false, + "description": "Network Time Protocol (NTP) Server" + } + }, + "description": "Check NTP port 123 is disabled and the device is not operating as an NTP server", + "expected_behavior": "The device dos not respond to NTP requests when it's IP is set as the NTP server on another device" + } + } + } + ] + } +} \ No newline at end of file diff --git a/test_orc/modules/nmap/nmap.Dockerfile b/test_orc/modules/nmap/nmap.Dockerfile new file mode 100644 index 000000000..12f23dde7 --- /dev/null +++ b/test_orc/modules/nmap/nmap.Dockerfile @@ -0,0 +1,11 @@ +# Image name: test-run/baseline-test +FROM test-run/base-test:latest + +# Copy over all configuration files +COPY modules/nmap/conf /testrun/conf + +# Load device binary files +COPY modules/nmap/bin /testrun/bin + +# Copy over all python files +COPY modules/nmap/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/nmap/python/src/nmap_module.py b/test_orc/modules/nmap/python/src/nmap_module.py new file mode 100644 index 000000000..7d5bd3604 --- /dev/null +++ b/test_orc/modules/nmap/python/src/nmap_module.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 + +import time +import util +import json +import threading +from test_module import TestModule + +LOG_NAME = "test_nmap" +LOGGER = None + + +class NmapModule(TestModule): + + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + self._unallowed_ports = [] + self._scan_tcp_results = None + self._udp_tcp_results = None + self._script_scan_results = None + global LOGGER + LOGGER = self._get_logger() + + def _security_nmap_ports(self, config): + LOGGER.info( + "Running security.nmap.ports test") + + # Delete the enabled key from the config if it exists + # to prevent it being treated as a test key + if "enabled" in config: + del config["enabled"] + + if self._device_ipv4_addr is not None: + # Run the monitor method asynchronously to keep this method non-blocking + self._tcp_scan_thread = threading.Thread( + target=self._scan_tcp_ports, args=(config,)) + self._udp_scan_thread = threading.Thread( + target=self._scan_udp_ports, args=(config,)) + self._script_scan_thread = threading.Thread( + target=self._scan_scripts, args=(config,)) + + self._tcp_scan_thread.daemon = True + self._udp_scan_thread.daemon = True + self._script_scan_thread.daemon = True + + self._tcp_scan_thread.start() + self._udp_scan_thread.start() + self._script_scan_thread.start() + + while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive() or self._script_scan_thread.is_alive(): + time.sleep(1) + + LOGGER.debug("TCP scan results: " + str(self._scan_tcp_results)) + LOGGER.debug("UDP scan results: " + str(self._scan_udp_results)) + LOGGER.debug("Service scan results: " + + str(self._script_scan_results)) + self._process_port_results( + tests=config) + LOGGER.info("Unallowed Ports: " + str(self._unallowed_ports)) + LOGGER.info("Script scan results:\n" + + json.dumps(self._script_scan_results)) + return len(self._unallowed_ports) == 0 + else: + LOGGER.info("Device ip address not resolved, skipping") + return None + + def _process_port_results(self, tests): + for test in tests: + LOGGER.info("Checking results for test: " + str(test)) + self._check_scan_results(test_config=tests[test]) + + def _check_scan_results(self, test_config): + port_config = {} + if "tcp_ports" in test_config: + port_config.update(test_config["tcp_ports"]) + elif "udp_ports" in test_config: + port_config.update(test_config["udp_ports"]) + + scan_results = {} + if self._scan_tcp_results is not None: + scan_results.update(self._scan_tcp_results) + if self._scan_udp_results is not None: + scan_results.update(self._scan_udp_results) + if self._script_scan_results is not None: + scan_results.update(self._script_scan_results) + if port_config is not None: + for port in port_config: + result = None + LOGGER.info("Checking port: " + str(port)) + LOGGER.debug("Port config: " + str(port_config[port])) + if port in scan_results: + if scan_results[port]["state"] == "open": + if not port_config[port]["allowed"]: + LOGGER.info("Unallowed port open") + self._unallowed_ports.append(str(port)) + result = False + else: + LOGGER.info("Allowed port open") + result = True + else: + LOGGER.info("Port is closed") + result = True + else: + LOGGER.info("Port not detected, closed") + result = True + + if result is not None: + port_config[port]["result"] = "compliant" if result else "non-compliant" + else: + port_config[port]["result"] = "skipped" + + def _scan_scripts(self, tests): + scan_results = {} + LOGGER.info("Checing for scan scripts") + for test in tests: + test_config = tests[test] + if "tcp_ports" in test_config: + for port in test_config["tcp_ports"]: + port_config = test_config["tcp_ports"][port] + if "service_scan" in port_config: + LOGGER.info("Service Scan Detected for: " + str(port)) + svc = port_config["service_scan"] + scan_results.update( + self._scan_tcp_with_script(svc["script"])) + if "udp_ports" in test_config: + for port in test_config["udp_ports"]: + if "service_scan" in port: + LOGGER.info("Service Scan Detected for: " + str(port)) + svc = port["service_scan"] + self._scan_udp_with_script(svc["script"], port) + scan_results.update( + self._scan_tcp_with_script(svc["script"])) + self._script_scan_results = scan_results + + def _scan_tcp_with_script(self, script_name, ports=None): + LOGGER.info("Running TCP nmap scan with script " + script_name) + scan_options = " -v -n T3 --host-timeout=6m -A --script " + script_name + port_options = " --open " + if ports is None: + port_options += " -p- " + else: + port_options += " -p" + ports + " " + results_file = "/runtime/output/" + self._module_name + "-"+script_name+".log" + nmap_options = scan_options + port_options + " -oG " + results_file + nmap_results, err = util.run_command( + "nmap " + nmap_options + " " + self._device_ipv4_addr) + LOGGER.info("Nmap TCP script scan complete") + LOGGER.info("nmap script results\n" + str(nmap_results)) + return self._process_nmap_results(nmap_results=nmap_results) + + def _scan_udp_with_script(self, script_name, ports=None): + LOGGER.info("Running UDP nmap scan with script " + script_name) + scan_options = " --sU -Pn -n --script " + script_name + port_options = " --open " + if ports is None: + port_options += " -p- " + else: + port_options += " -p" + ports + " " + nmap_options = scan_options + port_options + nmap_results, err = util.run_command( + "nmap " + nmap_options + self._device_ipv4_addr) + LOGGER.info("Nmap UDP script scan complete") + LOGGER.info("nmap script results\n" + str(nmap_results)) + return self._process_nmap_results(nmap_results=nmap_results) + + def _scan_tcp_ports(self, tests): + max_port = 1000 + ports = [] + for test in tests: + test_config = tests[test] + if "tcp_ports" in test_config: + for port in test_config["tcp_ports"]: + if int(port) > max_port: + ports.append(port) + ports_to_scan = "1-" + str(max_port) + if len(ports) > 0: + ports_to_scan += "," + ','.join(ports) + LOGGER.info("Running nmap TCP port scan") + LOGGER.info("TCP ports: " + str(ports_to_scan)) + nmap_results, err = util.run_command( + "nmap -sT -sV -Pn -v -p " + ports_to_scan + " --version-intensity 7 -T4 " + self._device_ipv4_addr) + LOGGER.info("TCP port scan complete") + self._scan_tcp_results = self._process_nmap_results( + nmap_results=nmap_results) + + def _scan_udp_ports(self, tests): + ports = [] + for test in tests: + test_config = tests[test] + if "udp_ports" in test_config: + for port in test_config["udp_ports"]: + ports.append(port) + if len(ports) > 0: + port_list = ','.join(ports) + LOGGER.info("Running nmap UDP port scan") + LOGGER.info("UDP ports: " + str(port_list)) + nmap_results, err = util.run_command( + "nmap -sU -sV -p " + port_list + " " + self._device_ipv4_addr) + LOGGER.info("UDP port scan complete") + self._scan_udp_results = self._process_nmap_results( + nmap_results=nmap_results) + + def _process_nmap_results(self, nmap_results): + results = {} + LOGGER.info("nmap results\n" + str(nmap_results)) + if nmap_results: + if "Service Info" in nmap_results: + rows = nmap_results.split("PORT")[1].split( + "Service Info")[0].split("\n") + elif "PORT" in nmap_results: + rows = nmap_results.split("PORT")[1].split( + "MAC Address")[0].split("\n") + if rows: + for result in rows[1:-1]: # Iterate skipping the header and tail rows + cols = result.split() + port = cols[0].split("/")[0] + # If results don't start with a a port number, it's likely a bleed over + # from previous result so we need to ignore it + if port.isdigit(): + version = "" + if len(cols) > 3: + # recombine full version information that may contain spaces + version = ' '.join(cols[3:]) + port_result = {cols[0].split( + "/")[0]: {"state": cols[1], "service": cols[2], "version": version}} + results.update(port_result) + return results diff --git a/test_orc/modules/nmap/python/src/run.py b/test_orc/modules/nmap/python/src/run.py new file mode 100644 index 000000000..4c8294769 --- /dev/null +++ b/test_orc/modules/nmap/python/src/run.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import argparse +import signal +import sys +import logger + +from nmap_module import NmapModule + +LOGGER = logger.get_logger('test_module') + +class NmapModuleRunner: + + def __init__(self,module): + + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) + + LOGGER.info("Starting nmap Module") + + self._test_module = NmapModule(module) + self._test_module.run_tests() + + def _handler(self, signum, *other): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping test module...") + LOGGER.info("Test module stopped") + sys.exit(1) + +def run(argv): + parser = argparse.ArgumentParser(description="Nmap Module Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument( + "-m", "--module", help="Define the module name to be used to create the log file") + + args = parser.parse_args() + + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + NmapModuleRunner(args.module.strip()) + +if __name__ == "__main__": + run(sys.argv) diff --git a/test_orc/python/src/module.py b/test_orc/python/src/module.py index 8121c34db..6b2f14f9d 100644 --- a/test_orc/python/src/module.py +++ b/test_orc/python/src/module.py @@ -15,9 +15,13 @@ class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-att container_name: str = None image_name :str = None enable_container: bool = True + network: bool = True timeout: int = 60 # Absolute path dir: str = None dir_name: str = None + + #Set IP Index for all test modules + ip_index: str = 9 diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index c257cd901..08c855d9a 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -19,9 +19,10 @@ class TestOrchestrator: """Manages and controls the test modules.""" - def __init__(self): + def __init__(self,net_orc): self._test_modules = [] self._module_config = None + self._net_orc = net_orc self._path = os.path.dirname(os.path.dirname( os.path.dirname(os.path.realpath(__file__)))) @@ -90,7 +91,9 @@ def _run_test_module(self, module, device): environment={ "HOST_USER": getpass.getuser(), "DEVICE_MAC": device.mac_addr, - "DEVICE_TEST_MODULES": device.test_modules + "DEVICE_TEST_MODULES": device.test_modules, + "IPV4_SUBNET": self._net_orc.network_config.ipv4_network, + "IPV6_SUBNET": self._net_orc.network_config.ipv6_network } ) except (docker.errors.APIError, docker.errors.ContainerError) as container_error: @@ -98,6 +101,11 @@ def _run_test_module(self, module, device): LOGGER.debug(container_error) return + # Mount the test container to the virtual network if requried + if module.network: + LOGGER.info("Mounting test module to the network") + self._net_orc._attach_test_module_to_network(module) + # Determine the module timeout time test_module_timeout = time.time() + module.timeout status = self._get_module_status(module)