diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 000000000..fbdbe442c --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,30 @@ +name: Testrun test suite + +on: + push: + pull_request: + schedule: + - cron: '0 13 * * *' + +jobs: + testrun: + name: Baseline + runs-on: ubuntu-20.04 + timeout-minutes: 20 + steps: + - name: Checkout source + uses: actions/checkout@v2.3.4 + - name: Run tests + shell: bash {0} + run: testing/test_baseline + + pylint: + name: Pylint + runs-on: ubuntu-20.04 + timeout-minutes: 20 + steps: + - name: Checkout source + uses: actions/checkout@v2.3.4 + - name: Run tests + shell: bash {0} + run: testing/test_pylint diff --git a/framework/test_runner.py b/framework/test_runner.py index 14cadf3e1..5c4bf1472 100644 --- a/framework/test_runner.py +++ b/framework/test_runner.py @@ -19,10 +19,12 @@ class TestRunner: - def __init__(self, config_file=None, validate=True, net_only=False): + def __init__(self, config_file=None, validate=True, net_only=False, single_intf=False): self._register_exits() self.test_run = TestRun(config_file=config_file, - validate=validate, net_only=net_only) + validate=validate, + net_only=net_only, + single_intf=single_intf) def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -57,6 +59,8 @@ def parse_args(argv): help="Turn off the validation of the network after network boot") parser.add_argument("-net", "--net-only", action="store_true", help="Run the network only, do not run tests") + parser.add_argument("--single-intf", action="store_true", + help="Single interface mode (experimental)") args, unknown = parser.parse_known_args() return args @@ -65,5 +69,6 @@ def parse_args(argv): args = parse_args(sys.argv) runner = TestRunner(config_file=args.config_file, validate=not args.no_validate, - net_only=args.net_only) + net_only=args.net_only, + single_intf=args.single_intf) runner.start() diff --git a/framework/testrun.py b/framework/testrun.py index 40076108b..55719d968 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -33,7 +33,7 @@ LOGGER = logger.get_logger('test_run') CONFIG_FILE = 'conf/system.json' EXAMPLE_CONFIG_FILE = 'conf/system.json.example' -RUNTIME = 300 +RUNTIME = 1500 LOCAL_DEVICES_DIR = 'local/devices' RESOURCE_DEVICES_DIR = 'resources/devices' @@ -51,9 +51,10 @@ class TestRun: # pylint: disable=too-few-public-methods orchestrator and user interface. """ - def __init__(self, config_file=CONFIG_FILE, validate=True, net_only=False): + def __init__(self, config_file=CONFIG_FILE, validate=True, net_only=False, single_intf=False): self._devices = [] self._net_only = net_only + self._single_intf = single_intf # Catch any exit signals self._register_exits() @@ -62,7 +63,10 @@ def __init__(self, config_file=CONFIG_FILE, validate=True, net_only=False): config_file_abs = self._get_config_abs(config_file=config_file) self._net_orc = net_orc.NetworkOrchestrator( - config_file=config_file_abs, validate=validate, async_monitor=not self._net_only) + config_file=config_file_abs, + validate=validate, + async_monitor=not self._net_only, + single_intf = self._single_intf) self._test_orc = test_orc.TestOrchestrator() def start(self): diff --git a/net_orc/python/src/network_orchestrator.py b/net_orc/python/src/network_orchestrator.py index 63391a24f..56ae93c3f 100644 --- a/net_orc/python/src/network_orchestrator.py +++ b/net_orc/python/src/network_orchestrator.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import getpass import ipaddress import json import os +import subprocess import sys import time import threading @@ -25,15 +27,16 @@ INTERNET_BRIDGE = "tr-c" PRIVATE_DOCKER_NET = "tr-private-net" CONTAINER_NAME = "network_orchestrator" -RUNTIME = 300 +RUNTIME = 1500 class NetworkOrchestrator: """Manage and controls a virtual testing network.""" - def __init__(self, config_file=CONFIG_FILE, validate=True, async_monitor=False): + def __init__(self, config_file=CONFIG_FILE, validate=True, async_monitor=False, single_intf = False): self._int_intf = None self._dev_intf = None + self._single_intf = single_intf self.listener = None @@ -153,6 +156,38 @@ def _ping(self, net_module): success = util.run_command(cmd, output=False) return success + def _ci_pre_network_create(self): + """ Stores network properties to restore network after + network creation and flushes internet interface + """ + + self._ethmac = subprocess.check_output( + f"cat /sys/class/net/{self._int_intf}/address", shell=True).decode("utf-8").strip() + self._gateway = subprocess.check_output( + "ip route | head -n 1 | awk '{print $3}'", shell=True).decode("utf-8").strip() + self._ipv4 = subprocess.check_output( + f"ip a show {self._int_intf} | grep \"inet \" | awk '{{print $2}}'", shell=True).decode("utf-8").strip() + self._ipv6 = subprocess.check_output( + f"ip a show {self._int_intf} | grep inet6 | awk '{{print $2}}'", shell=True).decode("utf-8").strip() + self._brd = subprocess.check_output( + f"ip a show {self._int_intf} | grep \"inet \" | awk '{{print $4}}'", shell=True).decode("utf-8").strip() + + def _ci_post_network_create(self): + """ Restore network connection in CI environment """ + LOGGER.info("post cr") + util.run_command(f"ip address del {self._ipv4} dev {self._int_intf}") + util.run_command(f"ip -6 address del {self._ipv6} dev {self._int_intf}") + util.run_command(f"ip link set dev {self._int_intf} address 00:B0:D0:63:C2:26") + util.run_command(f"ip addr flush dev {self._int_intf}") + util.run_command(f"ip addr add dev {self._int_intf} 0.0.0.0") + util.run_command(f"ip addr add dev {INTERNET_BRIDGE} {self._ipv4} broadcast {self._brd}") + util.run_command(f"ip -6 addr add {self._ipv6} dev {INTERNET_BRIDGE} ") + util.run_command(f"systemd-resolve --interface {INTERNET_BRIDGE} --set-dns 8.8.8.8") + util.run_command(f"ip link set dev {INTERNET_BRIDGE} up") + util.run_command(f"dhclient {INTERNET_BRIDGE}") + util.run_command(f"ip route del default via 10.1.0.1") + util.run_command(f"ip route add default via {self._gateway} src {self._ipv4[:-3]} metric 100 dev {INTERNET_BRIDGE}") + def _create_private_net(self): client = docker.from_env() try: @@ -186,6 +221,9 @@ def create_net(self): LOGGER.error("Configured interfaces are not ready for use. " + "Ensure both interfaces are connected.") sys.exit(1) + + if self._single_intf: + self._ci_pre_network_create() # Create data plane util.run_command("ovs-vsctl add-br " + DEVICE_BRIDGE) @@ -210,6 +248,9 @@ def create_net(self): util.run_command("ip link set dev " + DEVICE_BRIDGE + " up") util.run_command("ip link set dev " + INTERNET_BRIDGE + " up") + if self._single_intf: + self._ci_post_network_create() + self._create_private_net() self.listener = Listener(self._dev_intf) @@ -325,7 +366,7 @@ def _start_network_service(self, net_module): privileged=True, detach=True, mounts=net_module.mounts, - environment={"HOST_USER": os.getlogin()} + environment={"HOST_USER": getpass.getuser()} ) except docker.errors.ContainerError as error: LOGGER.error("Container run error") diff --git a/net_orc/python/src/network_validator.py b/net_orc/python/src/network_validator.py index 53fbcdbd0..2f01a06e9 100644 --- a/net_orc/python/src/network_validator.py +++ b/net_orc/python/src/network_validator.py @@ -5,6 +5,7 @@ import time import docker from docker.types import Mount +import getpass import logger import util @@ -144,7 +145,7 @@ def _start_network_device(self, device): privileged=True, detach=True, mounts=device.mounts, - environment={"HOST_USER": os.getlogin()} + environment={"HOST_USER": getpass.getuser()} ) except docker.errors.ContainerError as error: LOGGER.error("Container run error") diff --git a/test_orc/modules/baseline/python/src/run.py b/test_orc/modules/baseline/python/src/run.py index ffa171e17..8b55484ae 100644 --- a/test_orc/modules/baseline/python/src/run.py +++ b/test_orc/modules/baseline/python/src/run.py @@ -8,7 +8,7 @@ from baseline_module import BaselineModule LOGGER = logger.get_logger('test_module') -RUNTIME = 300 +RUNTIME = 1500 class BaselineModuleRunner: diff --git a/test_orc/modules/dns/python/src/run.py b/test_orc/modules/dns/python/src/run.py index 7ee5e7833..e5fedb67b 100644 --- a/test_orc/modules/dns/python/src/run.py +++ b/test_orc/modules/dns/python/src/run.py @@ -10,7 +10,7 @@ LOG_NAME = "dns_module" LOGGER = logger.get_logger(LOG_NAME) -RUNTIME = 300 +RUNTIME = 1500 class DNSModuleRunner: diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 85c6fb631..ee5cc5b45 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -1,4 +1,5 @@ """Provides high level management of the test orchestrator.""" +import getpass import os import json import time @@ -87,7 +88,7 @@ def _run_test_module(self, module, device): ), ], environment={ - "HOST_USER": os.getlogin(), + "HOST_USER": getpass.getuser(), "DEVICE_MAC": device.mac_addr, "DEVICE_TEST_MODULES": device.test_modules } diff --git a/testing/docker/ci_baseline/Dockerfile b/testing/docker/ci_baseline/Dockerfile new file mode 100644 index 000000000..7c3c1eebd --- /dev/null +++ b/testing/docker/ci_baseline/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:jammy + +#Update and get all additional requirements not contained in the base image +RUN apt-get update && apt-get -y upgrade + +RUN apt-get install -y isc-dhcp-client ntpdate coreutils moreutils inetutils-ping curl jq dnsutils + +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/testing/docker/ci_baseline/entrypoint.sh b/testing/docker/ci_baseline/entrypoint.sh new file mode 100755 index 000000000..bc2da3ec2 --- /dev/null +++ b/testing/docker/ci_baseline/entrypoint.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +OUT=/out/testrun_ci.json + +NTP_SERVER=10.10.10.5 +DNS_SERVER=10.10.10.4 + +function wout(){ + temp=${1//./\".\"} + key=${temp:1}\" + echo $key + value=$2 + jq "$key+=\"$value\"" $OUT | sponge $OUT +} + + +dig @8.8.8.8 +short www.google.com + +# DHCP +ip addr flush dev eth0 +PID_FILE=/var/run/dhclient.pid +if [ -f $PID_FILE ]; then + kill -9 $(cat $PID_FILE) || true + rm -f $PID_FILE +fi +dhclient -v eth0 + +echo "{}" > $OUT + +# Gen network +main_intf=$(ip route | grep '^default' | awk '{print $NF}') + +wout .network.main_intf $main_intf +wout .network.gateway $(ip route | head -n 1 | awk '{print $3}') +wout .network.ipv4 $(ip a show $main_intf | grep "inet " | awk '{print $2}') +wout .network.ipv6 $(ip a show $main_intf | grep inet6 | awk '{print $2}') +wout .network.ethmac $(cat /sys/class/net/$main_intf/address) + +wout .dns_response $(dig @$DNS_SERVER +short www.google.com | tail -1) +wout .ntp_offset $(ntpdate -q $NTP_SERVER | tail -1 | sed -E 's/.*offset ([-=0-9\.]*) sec/\1/') + +# INTERNET CONNECTION +google_com_response=$(curl -LI http://www.google.com -o /dev/null -w '%{http_code}\n' -s) +wout .network.internet $google_com_response + +# DHCP LEASE +while read pre name value; do + if [[ $pre != option ]]; then + continue; + fi + + wout .dhcp.$name $(echo "${value%;}" | tr -d '\"\\') + +done < <(grep -B 99 -m 1 "}" /var/lib/dhcp/dhclient.leases) + +cat $OUT \ No newline at end of file diff --git a/testing/test_baseline b/testing/test_baseline new file mode 100755 index 000000000..d7fc1e5c5 --- /dev/null +++ b/testing/test_baseline @@ -0,0 +1,73 @@ + +#!/bin/bash -e + +TESTRUN_OUT=/tmp/testrun.log + +# Setup requirements +sudo apt-get update +sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils + +pip3 install pytest + +# Setup device network +sudo ip link add dev endev0a type veth peer name endev0b +sudo ip link set dev endev0a up +sudo ip link set dev endev0b up +sudo docker network create -d macvlan -o parent=endev0b endev0 + +# Start OVS +sudo /usr/share/openvswitch/scripts/ovs-ctl start + +# Fix due to ordering +sudo docker build ./net_orc/ -t test-run/base -f net_orc/network/modules/base/base.Dockerfile + +# Build Test Container +sudo docker build ./testing/docker/ci_baseline -t ci1 -f ./testing/docker/ci_baseline/Dockerfile + +cat <conf/system.json +{ + "network": { + "device_intf": "endev0a", + "internet_intf": "eth0" + }, + "log_level": "DEBUG" +} +EOF + +sudo cmd/install + +sudo cmd/start --single-intf > $TESTRUN_OUT 2>&1 & +TPID=$! + +# Time to wait for testrun to be ready +WAITING=600 +for i in `seq 1 $WAITING`; do + if [[ -n $(fgrep "Waiting for devices on the network" $TESTRUN_OUT) ]]; then + break + fi + + if [[ ! -d /proc/$TPID ]]; then + cat $TESTRUN_OUT + echo "error encountered starting test run" + exit 1 + fi + + sleep 1 +done + +if [[ $i -eq $WAITING ]]; then + cat $TESTRUN_OUT + echo "failed after waiting $WAITING seconds for test-run start" + exit 1 +fi + +# Load Test Container +sudo docker run --network=endev0 --cap-add=NET_ADMIN -v /tmp:/out --privileged ci1 + +echo "Done baseline test" + +more $TESTRUN_OUT + +pytest testing/ + +exit $? diff --git a/testing/test_baseline.py b/testing/test_baseline.py new file mode 100644 index 000000000..3ab30a7c0 --- /dev/null +++ b/testing/test_baseline.py @@ -0,0 +1,49 @@ +import json +import pytest +import re +import os + +NTP_SERVER = '10.10.10.5' +DNS_SERVER = '10.10.10.4' + +CI_BASELINE_OUT = '/tmp/testrun_ci.json' + +@pytest.fixture +def container_data(): + dir = os.path.dirname(os.path.abspath(__file__)) + with open(CI_BASELINE_OUT) as f: + return json.load(f) + +@pytest.fixture +def validator_results(): + dir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(dir, '../', 'runtime/validation/faux-dev/result.json')) as f: + return json.load(f) + +def test_internet_connectivity(container_data): + assert container_data['network']['internet'] == 200 + +def test_dhcp_ntp_option(container_data): + """ Check DHCP gives NTP server as option """ + assert container_data['dhcp']['ntp-servers'] == NTP_SERVER + +def test_dhcp_dns_option(container_data): + assert container_data['dhcp']['domain-name-servers'] == DNS_SERVER + +def test_assigned_ipv4_address(container_data): + assert int(container_data['network']['ipv4'].split('.')[-1][:-3]) > 10 + +def test_ntp_server_reachable(container_data): + assert not 'no servers' in container_data['ntp_offset'] + +def test_dns_server_reachable(container_data): + assert not 'no servers' in container_data['dns_response'] + +def test_dns_server_resolves(container_data): + assert re.match(r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}', + container_data['dns_response']) + +def test_validator_results_compliant(validator_results): + results = [True if x['result'] == 'compliant' else False + for x in validator_results['results']] + assert all(results) diff --git a/testing/test_pylint b/testing/test_pylint new file mode 100755 index 000000000..833961d94 --- /dev/null +++ b/testing/test_pylint @@ -0,0 +1,26 @@ +#!/bin/bash + +ERROR_LIMIT=2534 + +sudo cmd/install + +source venv/bin/activate +sudo pip3 install pylint + +files=$(find . -path ./venv -prune -o -name '*.py' -print) + +OUT=pylint.out + +rm -f $OUT && touch $OUT +pylint $files -ry --extension-pkg-allow-list=docker 2>/dev/null | tee -a $OUT + +new_errors=$(cat $OUT | grep "statements analysed." | awk '{print $1}') + +echo "$new_errors > $ERROR_LIMIT?" +if (( $new_errors > $ERROR_LIMIT)); then + echo new errors $new_errors > error limit $ERROR_LIMIT + echo failing .. + exit 1 +fi + +exit 0