diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 87c8a814a..bf1d6ecc0 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -1,7 +1,6 @@
name: Testrun test suite
on:
- push:
pull_request:
schedule:
- cron: '0 13 * * *'
@@ -21,7 +20,6 @@ jobs:
testrun_tests:
name: Tests
runs-on: ubuntu-20.04
- needs: testrun_baseline
timeout-minutes: 40
steps:
- name: Checkout source
@@ -29,6 +27,27 @@ jobs:
- name: Run tests
shell: bash {0}
run: testing/tests/test_tests
+ - name: Archive runtime results
+ if: ${{ always() }}
+ run: sudo tar --exclude-vcs -czf runtime.tgz runtime/
+ - name: Upload runtime results
+ uses: actions/upload-artifact@v3
+ if: ${{ always() }}
+ with:
+ if-no-files-found: error
+ name: runtime_${{ github.workflow }}_${{ github.run_id }}
+ path: runtime.tgz
+
+ testrun_api:
+ name: API
+ runs-on: ubuntu-20.04
+ timeout-minutes: 40
+ steps:
+ - name: Checkout source
+ uses: actions/checkout@v2.3.4
+ - name: Run tests
+ shell: bash {0}
+ run: testing/api/test_api
pylint:
name: Pylint
diff --git a/bin/testrun b/bin/testrun
index 5fb8bf232..ea65d3565 100755
--- a/bin/testrun
+++ b/bin/testrun
@@ -15,7 +15,7 @@
# limitations under the License.
if [[ "$EUID" -ne 0 ]]; then
- echo "Must run as root. Use sudo testrun"
+ echo "Must run as root. Use sudo $0"
exit 1
fi
diff --git a/cmd/install b/cmd/install
index 6477b85fb..7997f37fa 100755
--- a/cmd/install
+++ b/cmd/install
@@ -24,4 +24,7 @@ pip3 install -r framework/requirements.txt
# required by python package weasyprint
sudo apt-get install libpangocairo-1.0-0
+#TODO move into docker build process
+(cd modules/ui && npm install && npm run build)
+
deactivate
\ No newline at end of file
diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py
index d3d0ca9f4..edf3ce5da 100644
--- a/framework/python/src/common/session.py
+++ b/framework/python/src/common/session.py
@@ -87,17 +87,21 @@ def _load_config(self):
if (NETWORK_KEY in config_file_json
and DEVICE_INTF_KEY in config_file_json.get(NETWORK_KEY)
and INTERNET_INTF_KEY in config_file_json.get(NETWORK_KEY)):
- self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY)
- self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY)
+ self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(
+ NETWORK_KEY, {}).get(DEVICE_INTF_KEY)
+ self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(
+ NETWORK_KEY, {}).get(INTERNET_INTF_KEY)
if RUNTIME_KEY in config_file_json:
self._config[RUNTIME_KEY] = config_file_json.get(RUNTIME_KEY)
if STARTUP_TIMEOUT_KEY in config_file_json:
- self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(STARTUP_TIMEOUT_KEY)
+ self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(
+ STARTUP_TIMEOUT_KEY)
if MONITOR_PERIOD_KEY in config_file_json:
- self._config[MONITOR_PERIOD_KEY] = config_file_json.get(MONITOR_PERIOD_KEY)
+ self._config[MONITOR_PERIOD_KEY] = config_file_json.get(
+ MONITOR_PERIOD_KEY)
if LOG_LEVEL_KEY in config_file_json:
self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY)
@@ -106,7 +110,8 @@ def _load_config(self):
self._config[API_PORT_KEY] = config_file_json.get(API_PORT_KEY)
if MAX_DEVICE_REPORTS_KEY in config_file_json:
- self._config[MAX_DEVICE_REPORTS_KEY] = config_file_json.get(MAX_DEVICE_REPORTS_KEY)
+ self._config[MAX_DEVICE_REPORTS_KEY] = config_file_json.get(
+ MAX_DEVICE_REPORTS_KEY)
def _save_config(self):
with open(self._config_file, 'w', encoding='utf-8') as f:
diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py
index d57db58cf..af05f2a2f 100644
--- a/framework/python/src/common/testreport.py
+++ b/framework/python/src/common/testreport.py
@@ -94,7 +94,7 @@ def to_pdf(self):
pdf_bytes = BytesIO()
HTML(string=report_html).write_pdf(pdf_bytes)
return pdf_bytes
-
+
def to_html(self):
json_data = self.to_json()
return f'''
@@ -118,18 +118,19 @@ def to_html(self):
'''
def generate_test_sections(self,json_data):
- results = json_data["tests"]["results"]
- sections = ""
+ results = json_data['tests']['results']
+ sections = ''
for result in results:
- sections += self.generate_test_section(result)
+ sections += self.generate_test_section(result)
return sections
def generate_test_section(self, result):
section_content = '\n'
for key, value in result.items():
- if value is not None: # Check if the value is not None
- formatted_key = key.replace('_', ' ').title() # Replace underscores and capitalize
- section_content += f'{formatted_key}: {value}
\n'
+ if value is not None: # Check if the value is not None
+ # Replace underscores and capitalize
+ formatted_key = key.replace('_', ' ').title()
+ section_content += f'{formatted_key}: {value}
\n'
section_content += '\n
\n'
return section_content
@@ -171,4 +172,4 @@ def generate_css(self):
text-decoration: none;
color: #007bff;
}
- '''
\ No newline at end of file
+ '''
diff --git a/framework/requirements.txt b/framework/requirements.txt
index 7141ae706..d833aca06 100644
--- a/framework/requirements.txt
+++ b/framework/requirements.txt
@@ -14,4 +14,8 @@ weasyprint
fastapi==0.99.1
psutil
uvicorn
-pydantic==1.10.11
\ No newline at end of file
+pydantic==1.10.11
+
+# Requirements for testing
+pytest
+pytest-timeout
\ No newline at end of file
diff --git a/modules/network/dhcp-1/dhcp-1.Dockerfile b/modules/network/dhcp-1/dhcp-1.Dockerfile
index 6b941d878..49845cc3b 100644
--- a/modules/network/dhcp-1/dhcp-1.Dockerfile
+++ b/modules/network/dhcp-1/dhcp-1.Dockerfile
@@ -25,7 +25,7 @@ RUN apt-get install -y wget
RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/
# Install dhcp server
-RUN apt-get install -y isc-dhcp-server radvd systemd
+RUN apt-get update && apt-get install -y isc-dhcp-server radvd systemd
# Copy over all configuration files
COPY $MODULE_DIR/conf /testrun/conf
diff --git a/modules/network/dhcp-2/dhcp-2.Dockerfile b/modules/network/dhcp-2/dhcp-2.Dockerfile
index 153aa50e7..e91465f36 100644
--- a/modules/network/dhcp-2/dhcp-2.Dockerfile
+++ b/modules/network/dhcp-2/dhcp-2.Dockerfile
@@ -25,7 +25,7 @@ RUN apt-get install -y wget
RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/
# Install dhcp server
-RUN apt-get install -y isc-dhcp-server radvd systemd
+RUN apt-get update && apt-get install -y isc-dhcp-server radvd systemd
# Copy over all configuration files
COPY $MODULE_DIR/conf /testrun/conf
diff --git a/modules/test/baseline/python/src/baseline_module.py b/modules/test/baseline/python/src/baseline_module.py
index 978f916fe..0e6222361 100644
--- a/modules/test/baseline/python/src/baseline_module.py
+++ b/modules/test/baseline/python/src/baseline_module.py
@@ -27,17 +27,17 @@ def __init__(self, module):
global LOGGER
LOGGER = self._get_logger()
- def _baseline_pass(self):
+ def _baseline_compliant(self):
LOGGER.info('Running baseline pass test')
LOGGER.info('Baseline pass test finished')
return True, 'Baseline pass test ran successfully'
- def _baseline_fail(self):
+ def _baseline_non_compliant(self):
LOGGER.info('Running baseline fail test')
LOGGER.info('Baseline fail test finished')
return False, 'Baseline fail test ran successfully'
- def _baseline_skip(self):
+ def _baseline_informational(self):
LOGGER.info('Running baseline skip test')
LOGGER.info('Baseline skip test finished')
return None, 'Baseline skip test ran successfully'
diff --git a/testing/api/mockito/get_devices.json b/testing/api/mockito/get_devices.json
new file mode 100644
index 000000000..2609eda35
--- /dev/null
+++ b/testing/api/mockito/get_devices.json
@@ -0,0 +1,46 @@
+[
+ {
+ "mac_addr": "00:1e:42:35:73:c4",
+ "manufacturer": "Teltonika",
+ "model": "TRB 140",
+ "test_modules": {
+ "dns": {
+ "enabled": false
+ },
+ "connection": {
+ "enabled": true
+ },
+ "ntp": {
+ "enabled": false
+ },
+ "baseline": {
+ "enabled": false
+ },
+ "nmap": {
+ "enabled": false
+ }
+ }
+ },
+ {
+ "mac_addr": "aa:bb:cc:dd:ee:ff",
+ "manufacturer": "Manufacturer X",
+ "model": "Device X",
+ "test_modules": {
+ "dns": {
+ "enabled": true
+ },
+ "connection": {
+ "enabled": true
+ },
+ "ntp": {
+ "enabled": true
+ },
+ "baseline": {
+ "enabled": false
+ },
+ "nmap": {
+ "enabled": true
+ }
+ }
+ }
+ ]
\ No newline at end of file
diff --git a/testing/api/mockito/invalid_request.json b/testing/api/mockito/invalid_request.json
new file mode 100644
index 000000000..5104263fd
--- /dev/null
+++ b/testing/api/mockito/invalid_request.json
@@ -0,0 +1,3 @@
+{
+ "error": "Invalid request received"
+}
\ No newline at end of file
diff --git a/testing/api/mockito/running_system_status.json b/testing/api/mockito/running_system_status.json
new file mode 100644
index 000000000..68a758b9c
--- /dev/null
+++ b/testing/api/mockito/running_system_status.json
@@ -0,0 +1,26 @@
+{
+ "status": "In Progress",
+ "device": {
+ "manufacturer": "Delta",
+ "model": "03-DIN-CPU",
+ "mac_addr": "01:02:03:04:05:06",
+ "firmware": "1.2.2"
+ },
+ "started": "2023-06-22T09:20:00.123Z",
+ "finished": null,
+ "tests": {
+ "total": 26,
+ "results": [
+ {
+ "name": "dns.network.hostname_resolution",
+ "description": "The device should resolve hostnames",
+ "result": "Compliant"
+ },
+ {
+ "name": "dns.network.from_dhcp",
+ "description": "The device should use the DNS server provided by the DHCP server",
+ "result": "Non-Compliant"
+ }
+ ]
+ }
+ }
\ No newline at end of file
diff --git a/testing/api/test_api b/testing/api/test_api
new file mode 100755
index 000000000..146b04156
--- /dev/null
+++ b/testing/api/test_api
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ifconfig
+
+# Setup requirements
+sudo apt-get update
+sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils isc-dhcp-client
+
+pip3 install pytest
+
+# Setup device network
+sudo ip link add dev dummynet type dummy
+sudo ip link add dev endev0a type veth peer name endev0b
+sudo ip link set dev endev0a up
+sudo ip link set dev endev0b up
+sudo docker network create -d macvlan -o parent=endev0b endev0
+
+sudo ip link add dev dummynet type dummy
+
+# Start OVS
+sudo /usr/share/openvswitch/scripts/ovs-ctl start
+
+# Build Test Container
+sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f ./testing/docker/ci_test_device1/Dockerfile
+
+sudo chown -R $USER local
+
+cat <local/system.json
+{
+ "network": {
+ "device_intf": "endev0a",
+ "internet_intf": "dummynet"
+ },
+ "log_level": "DEBUG"
+}
+EOF
+
+sudo cmd/install
+
+# Needs to be sudo because this invokes bin/testrun
+sudo venv/bin/python -m pytest -v testing/api/test_api.py
+
+exit $?
\ No newline at end of file
diff --git a/testing/api/test_api.py b/testing/api/test_api.py
new file mode 100644
index 000000000..f5f5b516b
--- /dev/null
+++ b/testing/api/test_api.py
@@ -0,0 +1,557 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test assertions for CI network baseline test"""
+# Temporarily disabled because using Pytest fixtures
+# TODO refactor fixtures to not trigger error
+# pylint: disable=redefined-outer-name
+
+from collections.abc import Awaitable, Callable
+import copy
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import shutil
+import signal
+import subprocess
+import time
+from typing import Iterator
+import pytest
+import requests
+
+ALL_DEVICES = "*"
+API = "http://127.0.0.1:8000"
+LOG_PATH = "/tmp/testrun.log"
+TEST_SITE_DIR = ".."
+
+DEVICES_DIRECTORY = "../../local/devices"
+TESTING_DEVICES = "../device_configs/"
+SYSTEM_CONFIG_PATH = "../../local/system.json"
+
+BASELINE_MAC_ADDR = "02:42:aa:00:01:01"
+ALL_MAC_ADDR = "02:42:aa:00:00:01"
+
+
+def pretty_print(dictionary: dict):
+ """ Pretty print dictionary """
+ print(json.dumps(dictionary, indent=4))
+
+
+def query_system_status() -> str:
+ """Query system status from API and returns this"""
+ r = requests.get(f"{API}/system/status")
+ response = json.loads(r.text)
+ return response["status"]
+
+
+def query_test_count() -> int:
+ """Queries status and returns number of test results"""
+ r = requests.get(f"{API}/system/status")
+ response = json.loads(r.text)
+ return len(response["tests"]["results"])
+
+
+def start_test_device(
+ device_name, mac_address, image_name="ci_test_device1", args=""
+):
+ """ Start test device container with given name """
+ cmd = subprocess.run(
+ f"docker run -d --network=endev0 --mac-address={mac_address}"
+ f" --cap-add=NET_ADMIN -v /tmp:/out --privileged --name={device_name}"
+ f" {image_name} {args}",
+ shell=True,
+ check=True,
+ capture_output=True,
+ )
+ print(cmd.stdout)
+
+
+def stop_test_device(device_name):
+ """ Stop docker container with given name """
+ cmd = subprocess.run(
+ f"docker stop {device_name}", shell=True, capture_output=True
+ )
+ print(cmd.stdout)
+ cmd = subprocess.run(
+ f"docker rm {device_name}", shell=True, capture_output=True
+ )
+ print(cmd.stdout)
+
+
+def docker_logs(device_name):
+ """ Print docker logs from given docker container name """
+ cmd = subprocess.run(
+ f"docker logs {device_name}", shell=True, capture_output=True
+ )
+ print(cmd.stdout)
+
+
+@pytest.fixture
+def empty_devices_dir():
+ """ Use e,pty devices directory """
+ local_delete_devices(ALL_DEVICES)
+
+
+@pytest.fixture
+def testing_devices():
+ """ Use devices from the testing/device_configs directory """
+ local_delete_devices(ALL_DEVICES)
+ shutil.copytree(
+ os.path.join(os.path.dirname(__file__), TESTING_DEVICES),
+ os.path.join(os.path.dirname(__file__), DEVICES_DIRECTORY),
+ dirs_exist_ok=True,
+ )
+ return local_get_devices()
+
+
+@pytest.fixture
+def testrun(request):
+ """ Start intstance of testrun """
+ test_name = request.node.originalname
+ proc = subprocess.Popen(
+ "bin/testrun",
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding="utf-8",
+ preexec_fn=os.setsid,
+ )
+
+ while True:
+ try:
+ outs, errs = proc.communicate(timeout=1)
+ except subprocess.TimeoutExpired as e:
+ if e.output is not None:
+ output = e.output.decode("utf-8")
+ if re.search("API waiting for requests", output):
+ break
+ except Exception as e:
+ pytest.fail("testrun terminated")
+
+ time.sleep(2)
+
+ yield
+
+ os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+ try:
+ outs, errs = proc.communicate(timeout=60)
+ except Exception as e:
+ print(e.output)
+ os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+ pytest.exit(
+ "waited 60s but test run did not cleanly exit .. terminating all tests"
+ )
+
+ print(outs)
+
+ cmd = subprocess.run(
+ f"docker stop $(docker ps -a -q)", shell=True, capture_output=True
+ )
+ print(cmd.stdout)
+ cmd = subprocess.run(
+ f"docker rm $(docker ps -a -q)", shell=True, capture_output=True
+ )
+ print(cmd.stdout)
+
+
+def until_true(func: Callable, message: str, timeout: int):
+ """ Blocks until given func returns True
+
+ Raises:
+ Exception if timeout has elapsed
+ """
+ expiry_time = time.time() + timeout
+ while time.time() < expiry_time:
+ if func():
+ return True
+ time.sleep(1)
+ raise Exception(f"Timed out waiting {timeout}s for {message}")
+
+
+def dict_paths(thing: dict, stem: str = "") -> Iterator[str]:
+ """Returns json paths (in dot notation) from a given dictionary"""
+ for k, v in thing.items():
+ path = f"{stem}.{k}" if stem else k
+ if isinstance(v, dict):
+ yield from dict_paths(v, path)
+ else:
+ yield path
+
+
+def get_network_interfaces():
+ """return list of network interfaces on machine
+
+ uses /sys/class/net rather than inetfaces as test-run uses the latter
+ """
+ path = Path("/sys/class/net")
+ return [i.stem for i in path.iterdir() if i.is_dir()]
+
+
+def local_delete_devices(path):
+ """ Deletes all local devices
+ """
+ devices_path = os.path.join(os.path.dirname(__file__), DEVICES_DIRECTORY)
+ for thing in Path(devices_path).glob(path):
+ if thing.is_file():
+ thing.unlink()
+ else:
+ shutil.rmtree(thing)
+
+
+def local_get_devices():
+ """ Returns path to device configs of devices in local/devices directory"""
+ return sorted(
+ Path(os.path.join(os.path.dirname(__file__), DEVICES_DIRECTORY)).glob(
+ "*/device_config.json"
+ )
+ )
+
+
+def test_get_system_interfaces(testrun):
+ """Tests API system interfaces against actual local interfaces"""
+ r = requests.get(f"{API}/system/interfaces")
+ response = json.loads(r.text)
+ local_interfaces = get_network_interfaces()
+ assert set(response) == set(local_interfaces)
+
+ # schema expects a flat list
+ assert all([isinstance(x, str) for x in response])
+
+
+def test_modify_device(testing_devices, testrun):
+ with open(
+ os.path.join(
+ os.path.dirname(__file__), DEVICES_DIRECTORY, testing_devices[1]
+ )
+ ) as f:
+ local_device = json.load(f)
+
+ mac_addr = local_device["mac_addr"]
+ new_model = "Alphabet"
+
+ r = requests.get(f"{API}/devices")
+ all_devices = json.loads(r.text)
+
+ api_device = next(x for x in all_devices if x["mac_addr"] == mac_addr)
+
+ updated_device = copy.deepcopy(api_device)
+ updated_device["model"] = new_model
+
+ new_test_modules = {
+ k: {"enabled": not v["enabled"]}
+ for k, v in updated_device["test_modules"].items()
+ }
+ updated_device["test_modules"] = new_test_modules
+
+ print("updated_device")
+ pretty_print(updated_device)
+ print("api_device")
+ pretty_print(api_device)
+
+ # update device
+ r = requests.post(f"{API}/device", data=json.dumps(updated_device))
+
+ assert r.status_code == 200
+
+ r = requests.get(f"{API}/devices")
+ all_devices = json.loads(r.text)
+ updated_device_api = next(x for x in all_devices if x["mac_addr"] == mac_addr)
+
+ assert updated_device_api["model"] == new_model
+ assert updated_device_api["test_modules"] == new_test_modules
+
+
+
+def test_create_get_devices(empty_devices_dir, testrun):
+ device_1 = {
+ "manufacturer": "Google",
+ "model": "First",
+ "mac_addr": "00:1e:42:35:73:c4",
+ "test_modules": {
+ "dns": {"enabled": True},
+ "connection": {"enabled": True},
+ "ntp": {"enabled": True},
+ "baseline": {"enabled": True},
+ "nmap": {"enabled": True},
+ },
+ }
+
+ r = requests.post(f"{API}/device", data=json.dumps(device_1))
+ print(r.text)
+ device1_response = r.text
+ assert r.status_code == 201
+ assert len(local_get_devices()) == 1
+
+ device_2 = {
+ "manufacturer": "Google",
+ "model": "Second",
+ "mac_addr": "00:1e:42:35:73:c6",
+ "test_modules": {
+ "dns": {"enabled": True},
+ "connection": {"enabled": True},
+ "ntp": {"enabled": True},
+ "baseline": {"enabled": True},
+ "nmap": {"enabled": True},
+ },
+ }
+ r = requests.post(f"{API}/device", data=json.dumps(device_2))
+ device2_response = json.loads(r.text)
+ assert r.status_code == 201
+ assert len(local_get_devices()) == 2
+
+ # Test that returned devices API endpoint matches expected structure
+ r = requests.get(f"{API}/devices")
+ all_devices = json.loads(r.text)
+ pretty_print(all_devices)
+
+ with open(
+ os.path.join(os.path.dirname(__file__), "mockito/get_devices.json")
+ ) as f:
+ mockito = json.load(f)
+
+ print(mockito)
+
+ # Validate structure
+ assert all([isinstance(x, dict) for x in all_devices])
+
+ # TOOO uncomment when is done
+ # assert set(dict_paths(mockito[0])) == set(dict_paths(all_devices[0]))
+
+ # Validate contents of given keys matches
+ for key in ["mac_addr", "manufacturer", "model"]:
+ assert set([all_devices[0][key], all_devices[1][key]]) == set(
+ [device_1[key], device_2[key]]
+ )
+
+
+def test_get_system_config(testrun):
+ r = requests.get(f"{API}/system/config")
+
+ with open(os.path.join(os.path.dirname(__file__), SYSTEM_CONFIG_PATH)) as f:
+ local_config = json.load(f)
+
+ api_config = json.loads(r.text)
+
+ # validate structure
+ assert set(dict_paths(api_config)) | set(dict_paths(local_config)) == set(
+ dict_paths(api_config)
+ )
+
+ assert (
+ local_config["network"]["device_intf"]
+ == api_config["network"]["device_intf"]
+ )
+ assert (
+ local_config["network"]["internet_intf"]
+ == api_config["network"]["internet_intf"]
+ )
+
+
+# TODO change to invalid jsdon request
+@pytest.mark.skip()
+def test_invalid_path_get(testrun):
+ r = requests.get(f"{API}/blah/blah")
+ response = json.loads(r.text)
+ assert r.status_code == 404
+ with open(
+ os.path.join(os.path.dirname(__file__), "mockito/invalid_request.json")
+ ) as f:
+ mockito = json.load(f)
+
+ # validate structure
+ assert set(dict_paths(mockito)) == set(dict_paths(response))
+
+
+def test_trigger_run(testing_devices, testrun):
+ payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}}
+ r = requests.post(f"{API}/system/start", data=json.dumps(payload))
+ assert r.status_code == 200
+
+ until_true(
+ lambda: query_system_status().lower() == "waiting for device",
+ "system status is `waiting for device`",
+ 30,
+ )
+
+ start_test_device("x123", BASELINE_MAC_ADDR)
+
+ until_true(
+ lambda: query_system_status().lower() == "compliant",
+ "system status is `complete`",
+ 900,
+ )
+
+ stop_test_device("x123")
+
+ # Validate response
+ r = requests.get(f"{API}/system/status")
+ response = json.loads(r.text)
+ pretty_print(response)
+
+ # Validate results
+ results = {x["name"]: x for x in response["tests"]["results"]}
+ print(results)
+ # there are only 3 baseline tests
+ assert len(results) == 3
+
+ # Validate structure
+ with open(
+ os.path.join(
+ os.path.dirname(__file__), "mockito/running_system_status.json"
+ )
+ ) as f:
+ mockito = json.load(f)
+
+ # validate structure
+ assert set(dict_paths(mockito)).issubset(set(dict_paths(response)))
+
+ # Validate results structure
+ assert set(dict_paths(mockito["tests"]["results"][0])).issubset(
+ set(dict_paths(response["tests"]["results"][0]))
+ )
+
+ # Validate a result
+ assert results["baseline.compliant"]["result"] == "Compliant"
+
+def test_stop_running_test(testing_devices, testrun):
+ payload = {"device": {"mac_addr": ALL_MAC_ADDR, "firmware": "asd"}}
+ r = requests.post(f"{API}/system/start", data=json.dumps(payload))
+ assert r.status_code == 200
+
+ until_true(
+ lambda: query_system_status().lower() == "waiting for device",
+ "system status is `waiting for device`",
+ 30,
+ )
+
+ start_test_device("x12345", ALL_MAC_ADDR)
+
+ until_true(
+ lambda: query_test_count() > 1,
+ "system status is `complete`",
+ 1000,
+ )
+
+ stop_test_device("x12345")
+
+ # Validate response
+ r = requests.post(f"{API}/system/stop")
+ response = json.loads(r.text)
+ pretty_print(response)
+ assert response == {"success": "Test Run stopped"}
+ time.sleep(1)
+ # Validate response
+ r = requests.get(f"{API}/system/status")
+ response = json.loads(r.text)
+ pretty_print(response)
+
+ #TODO uncomment when bug is fixed
+ #assert len(response["tests"]["results"]) == response["tests"]["total"]
+ assert len(response["tests"]["results"]) < 15
+ #TODO uncomment when bug is fixed
+ #assert response["status"] == "Stopped"
+
+
+@pytest.mark.skip()
+def test_stop_running_not_running(testrun):
+ # Validate response
+ r = requests.post(f"{API}/system/stop")
+ response = json.loads(r.text)
+ pretty_print(response)
+
+ assert False
+
+# TODO enable test because functionality is broken
+@pytest.mark.skip()
+def test_multiple_runs(testing_devices, testrun):
+ payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}}
+ r = requests.post(f"{API}/system/start", data=json.dumps(payload))
+ assert r.status_code == 200
+ print(r.text)
+
+ until_true(
+ lambda: query_system_status().lower() == "waiting for device",
+ "system status is `waiting for device`",
+ 30,
+ )
+
+ start_test_device("x123", BASELINE_MAC_ADDR)
+
+ until_true(
+ lambda: query_system_status().lower() == "compliant",
+ "system status is `complete`",
+ 900,
+ )
+
+ stop_test_device("x123")
+
+ # Validate response
+ r = requests.get(f"{API}/system/status")
+ response = json.loads(r.text)
+ pretty_print(response)
+
+ # Validate results
+ results = {x["name"]: x for x in response["tests"]["results"]}
+ print(results)
+ # there are only 3 baseline tests
+ assert len(results) == 3
+
+ payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}}
+ r = requests.post(f"{API}/system/start", data=json.dumps(payload))
+ # assert r.status_code == 200
+ # returns 409
+ print(r.text)
+
+ until_true(
+ lambda: query_system_status().lower() == "waiting for device",
+ "system status is `waiting for device`",
+ 30,
+ )
+
+ start_test_device("x123", BASELINE_MAC_ADDR)
+
+ until_true(
+ lambda: query_system_status().lower() == "compliant",
+ "system status is `complete`",
+ 900,
+ )
+
+ stop_test_device("x123")
+
+#TODO uncomment when functionality is implemented
+@pytest.mark.skip()
+def test_create_invalid_chars(empty_devices_dir, testrun):
+ # local_delete_devices(ALL_DEVICES)
+ # We must start test run with no devices in local/devices for this test to function as expected!
+ assert len(local_get_devices()) == 0
+
+ # Test adding device
+ device_1 = {
+ "manufacturer": "/'disallowed characters///",
+ "model": "First",
+ "mac_addr": BASELINE_MAC_ADDR,
+ "test_modules": {
+ "dns": {"enabled": False},
+ "connection": {"enabled": True},
+ "ntp": {"enabled": True},
+ "baseline": {"enabled": True},
+ "nmap": {"enabled": True},
+ },
+ }
+
+ r = requests.post(f"{API}/device", data=json.dumps(device_1))
+ print(r.text)
+ print(r.status_code)
+
diff --git a/testing/baseline/test_baseline b/testing/baseline/test_baseline
index 61d0f9b56..196b5f280 100755
--- a/testing/baseline/test_baseline
+++ b/testing/baseline/test_baseline
@@ -52,7 +52,7 @@ sudo bin/testrun --single-intf --no-ui > $TESTRUN_OUT 2>&1 &
TPID=$!
# Time to wait for testrun to be ready
-WAITING=600
+WAITING=750
for i in `seq 1 $WAITING`; do
if [[ -n $(fgrep "Waiting for devices on the network" $TESTRUN_OUT) ]]; then
break
diff --git a/testing/device_configs/only_baseline/device_config.json b/testing/device_configs/only_baseline/device_config.json
new file mode 100644
index 000000000..925929f81
--- /dev/null
+++ b/testing/device_configs/only_baseline/device_config.json
@@ -0,0 +1,25 @@
+{
+ "manufacturer": "Google",
+ "model": "Baseline",
+ "mac_addr": "02:42:aa:00:01:01",
+ "test_modules": {
+ "dns": {
+ "enabled": false
+ },
+ "connection": {
+ "enabled": false
+ },
+ "ntp": {
+ "enabled": false
+ },
+ "baseline": {
+ "enabled": true
+ },
+ "nmap": {
+ "enabled": false
+ },
+ "tls": {
+ "enabled": false
+ }
+ }
+}
diff --git a/testing/device_configs/tester1/device_config.json b/testing/device_configs/tester1/device_config.json
index 268399b72..55a5036ca 100644
--- a/testing/device_configs/tester1/device_config.json
+++ b/testing/device_configs/tester1/device_config.json
@@ -4,16 +4,16 @@
"mac_addr": "02:42:aa:00:00:01",
"test_modules": {
"dns": {
- "enabled": false
+ "enabled": true
},
"connection": {
- "enabled": false
+ "enabled": true
},
"ntp": {
- "enabled": false
+ "enabled": true
},
"baseline": {
- "enabled": false
+ "enabled": true
},
"nmap": {
"enabled": true
diff --git a/testing/device_configs/tester2/device_config.json b/testing/device_configs/tester2/device_config.json
index 8b090d80a..b037feb6d 100644
--- a/testing/device_configs/tester2/device_config.json
+++ b/testing/device_configs/tester2/device_config.json
@@ -4,16 +4,16 @@
"mac_addr": "02:42:aa:00:00:02",
"test_modules": {
"dns": {
- "enabled": false
+ "enabled": true
},
"connection": {
- "enabled": false
+ "enabled": true
},
"ntp": {
"enabled": true
},
"baseline": {
- "enabled": false
+ "enabled": true
},
"nmap": {
"enabled": true
diff --git a/testing/device_configs/tester3/device_config.json b/testing/device_configs/tester3/device_config.json
new file mode 100644
index 000000000..b7792027e
--- /dev/null
+++ b/testing/device_configs/tester3/device_config.json
@@ -0,0 +1,22 @@
+{
+ "manufacturer": "Google",
+ "model": "Tester 3",
+ "mac_addr": "02:42:aa:00:00:03",
+ "test_modules": {
+ "dns": {
+ "enabled": false
+ },
+ "connection": {
+ "enabled": true
+ },
+ "ntp": {
+ "enabled": false
+ },
+ "baseline": {
+ "enabled": true
+ },
+ "nmap": {
+ "enabled": false
+ }
+ }
+}
diff --git a/testing/docker/ci_test_device1/Dockerfile b/testing/docker/ci_test_device1/Dockerfile
index a362e2a4d..1c62d231d 100644
--- a/testing/docker/ci_test_device1/Dockerfile
+++ b/testing/docker/ci_test_device1/Dockerfile
@@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
# Update and get all additional requirements not contained in the base image
RUN apt-get update && apt-get -y upgrade
-RUN apt-get update && apt-get install -y isc-dhcp-client ntpdate coreutils moreutils inetutils-ping curl jq dnsutils openssl netcat-openbsd
+RUN apt-get update && apt-get install -y isc-dhcp-client ntpdate coreutils moreutils inetutils-ping curl jq dnsutils openssl netcat-openbsd arping
COPY entrypoint.sh /entrypoint.sh
diff --git a/testing/docker/ci_test_device1/entrypoint.sh b/testing/docker/ci_test_device1/entrypoint.sh
index 9152af0c8..dee51d50e 100755
--- a/testing/docker/ci_test_device1/entrypoint.sh
+++ b/testing/docker/ci_test_device1/entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
ip a
@@ -17,6 +17,7 @@ OUT=/out/testrun_ci.json
NTP_SERVER=10.10.10.5
DNS_SERVER=10.10.10.4
+INTF=eth0
function wout(){
temp=${1//./\".\"}
@@ -30,13 +31,15 @@ function wout(){
dig @8.8.8.8 +short www.google.com
# DHCP
-ip addr flush dev eth0
+ip addr flush dev $INTF
PID_FILE=/var/run/dhclient.pid
if [ -f $PID_FILE ]; then
kill -9 $(cat $PID_FILE) || true
rm -f $PID_FILE
fi
-dhclient -v eth0
+dhclient -v $INTF
+DHCP_TPID=$!
+echo $DHCP_TPID
if [ -n "${options[oddservices]}" ]; then
@@ -108,4 +111,37 @@ if [ -n "${options[ntpv3_time_google_com]}" ]; then
done) &
fi
+if [ -n "${options[dns_google]}" ]; then
+ echo starting dns requests to 8.8.8.8
+ (while true; do dig @8.8.8.8 +short www.google.com; sleep 3; done) &
+fi
+
+if [ -n "${options[dns_dhcp]}" ]; then
+ echo starting dns requests to $DNS_SERVER
+ (while true; do dig @$DNS_SERVER +short www.google.com; sleep 3; done) &
+fi
+
+if [ -n "${options[kill_dhcp]}" ]; then
+ echo killing DHCP
+ ipv4=$(ip a show $INTF | grep "inet " | awk '{print $2}')
+ pkill -f dhclient
+ ip addr change $ipv4 dev $INTF valid_lft forever preferred_lft forever
+fi
+
+if [ -n "${options[request_fixed]}" ]; then
+ ipv4=$(ip a show $INTF | grep "inet " | awk '{print $2}')
+
+ cat <>/etc/dhcp/dhclient.conf
+interface "$INTF" {
+ send dhcp-requested-address ${ipv4%\/*}
+}
+EOF
+dhclient -v $INTF
+
+fi
+
+
+
+(while true; do arping 10.10.10.1; sleep 10; done) &
+(while true; do ip a | cat; sleep 10; done) &
tail -f /dev/null
\ No newline at end of file
diff --git a/testing/pylint/test_pylint b/testing/pylint/test_pylint
index 3f4d8a3ed..6d28226cc 100755
--- a/testing/pylint/test_pylint
+++ b/testing/pylint/test_pylint
@@ -21,7 +21,7 @@ sudo cmd/install
source venv/bin/activate
sudo pip3 install pylint
-files=$(find . -path ./venv -prune -o -name '*.py' -print)
+files=$(find ./framework -path ./venv -prune -o -name '*.py' -print)
OUT=pylint.out
diff --git a/testing/tests/test_tests b/testing/tests/test_tests
index 04f76daee..9f997f10c 100755
--- a/testing/tests/test_tests
+++ b/testing/tests/test_tests
@@ -19,6 +19,7 @@ ip a
TEST_DIR=/tmp/results
MATRIX=testing/tests/test_tests.json
+rm -rf $TEST_DIR/
mkdir -p $TEST_DIR
# Setup requirements
@@ -29,27 +30,33 @@ pip3 install pytest
# Start OVS
# Setup device network
+sudo ip link add dev xyz type dummy
sudo ip link add dev endev0a type veth peer name endev0b
sudo ip link set dev endev0a up
sudo ip link set dev endev0b up
-sudo docker network create -d macvlan -o parent=endev0b endev1
+sudo docker network remove endev0
+sudo docker network create -d macvlan -o parent=endev0b endev0
sudo /usr/share/openvswitch/scripts/ovs-ctl start
# Build Test Container
sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f ./testing/docker/ci_test_device1/Dockerfile
+sudo chown -R $USER local
+
cat <local/system.json
{
"network": {
"device_intf": "endev0a",
- "internet_intf": "eth0"
+ "internet_intf": "xyz"
},
"log_level": "DEBUG",
"monitor_period": 30
}
EOF
+cat local/system.json
+
mkdir -p local/devices
cp -r testing/device_configs/* local/devices
@@ -57,6 +64,9 @@ sudo cmd/install
TESTERS=$(jq -r 'keys[]' $MATRIX)
for tester in $TESTERS; do
+ if [ -n $1 ] && [ $1 != $tester ]; then
+ continue
+ fi
testrun_log=$TEST_DIR/${tester}_testrun.log
device_log=$TEST_DIR/${tester}_device.log
@@ -65,11 +75,11 @@ for tester in $TESTERS; do
args=$(jq -r .$tester.args $MATRIX)
touch $testrun_log
- sudo timeout 900 bin/testrun --single-intf --no-ui --no-validate > $testrun_log 2>&1 &
+ sudo timeout 1800 bin/testrun --single-intf --no-ui --no-validate > $testrun_log 2>&1 &
TPID=$!
# Time to wait for testrun to be ready
- WAITING=600
+ WAITING=800
for i in `seq 1 $WAITING`; do
tail -1 $testrun_log
if [[ -n $(fgrep "Waiting for devices on the network" $testrun_log) ]]; then
@@ -91,9 +101,12 @@ for tester in $TESTERS; do
exit 1
fi
+ # helps unclean exits when running locally
+ sudo docker stop $tester && sudo docker rm $tester
+
# Load Test Container
sudo docker run -d \
- --network=endev1 \
+ --network=endev0 \
--mac-address=$ethmac \
--cap-add=NET_ADMIN \
-v /tmp:/out \
@@ -101,20 +114,35 @@ for tester in $TESTERS; do
--name=$tester \
ci_test_device1 $args
- wait $TPID
+wait $TPID
+#WAITING=600
+#for i in `seq 1 $WAITING`; do
+# tail -1 $testrun_log
+# if [[ -n $(fgrep "All tests complete" $testrun_log) ]]; then
+# sleep 10
+# kill -9 $TPID
+# fi
+#
+# if [[ ! -d /proc/$TPID ]]; then
+# break
+# fi
+#
+# sleep 1
+# done
+
+
# Following line indicates that tests are completed but wait till it exits
# Completed running test modules on device with mac addr 7e:41:12:d2:35:6a
#Change this line! - LOGGER.info(f"""Completed running test modules on device
# with mac addr {device.mac_addr}""")
- ls runtime
- more runtime/network/*.log
- sudo docker kill $tester
+ #more runtime/network/*.log | cat
sudo docker logs $tester | cat
-
+ sudo docker kill $tester && sudo docker rm $tester
+
cp runtime/test/${ethmac//:/}/report.json $TEST_DIR/$tester.json
- more $TEST_DIR/$tester.json
- more $testrun_log
+ more $TEST_DIR/$tester.json | cat
+ more $testrun_log | cat
done
diff --git a/testing/tests/test_tests.json b/testing/tests/test_tests.json
index 7d2b88678..728f7764f 100644
--- a/testing/tests/test_tests.json
+++ b/testing/tests/test_tests.json
@@ -1,21 +1,35 @@
{
"tester1": {
"image": "test-run/ci_test1",
- "args": "oddservices",
+ "args": "oddservices dns_static",
"ethmac": "02:42:aa:00:00:01",
"expected_results": {
"security.nmap.ports": "Non-Compliant"
}
},
"tester2": {
+ "description": "expected to pass most things",
"image": "test-run/ci_test1",
- "args": "ntpv4_dhcp",
+ "args": "ntpv4_dhcp dns_dhcp",
"ethmac": "02:42:aa:00:00:02",
"expected_results": {
"security.nmap.ports": "Compliant",
"ntp.network.ntp_support": "Compliant",
- "ntp.network.ntp_dhcp": "Compliant"
+ "ntp.network.ntp_dhcp": "Compliant",
+ "connection.shared_address": "Compliant",
+ "connection.dhcp_address": "Compliant",
+ "connection.mac_address": "Compliant",
+ "connection.target_ping": "Compliant",
+ "connection.single_ip": "Compliant",
+ "connection.ipaddr.ip_change": "Compliant"
}
+ },
+ "tester3": {
+ "description": "",
+ "image": "test-run/ci_test1",
+ "args": "kill_dhcp",
+ "ethmac": "02:42:aa:00:00:03",
+ "expected_results": {}
}
}
\ No newline at end of file
diff --git a/testing/tests/test_tests.py b/testing/tests/test_tests.py
index 666e65783..a14afb2cb 100644
--- a/testing/tests/test_tests.py
+++ b/testing/tests/test_tests.py
@@ -46,10 +46,8 @@ def collect_expected_results(expected_results):
def collect_actual_results(results_dict):
""" Yields results from an already loaded testrun results file """
# "module"."results".[list]."result"
- for maybe_module, child in results_dict.items():
- if 'results' in child and maybe_module != 'baseline':
- for test in child['results']:
- yield TestResult(test['name'], test['result'])
+ for test in results_dict.get('tests', {}).get('results', []):
+ yield TestResult(test['name'], test['result'])
@pytest.fixture
@@ -73,8 +71,7 @@ def test_tests(results, test_matrix):
for tester, props in test_matrix.items():
expected = set(collect_expected_results(props['expected_results']))
actual = set(collect_actual_results(results[tester]))
-
- assert expected.issubset(actual), f'{tester} expected results not obtained'
+ assert expected & actual == expected
def test_list_tests(capsys, results, test_matrix):
all_tests = set(itertools.chain.from_iterable(
@@ -95,7 +92,7 @@ def test_list_tests(capsys, results, test_matrix):
print('============')
print('============')
print('tests seen:')
- print('\n'.join([x.name for x in all_tests]))
+ print('\n'.join(set([x.name for x in all_tests])))
print('\ntesting for pass:')
print('\n'.join(ci_pass))
print('\ntesting for fail:')
@@ -103,7 +100,14 @@ def test_list_tests(capsys, results, test_matrix):
print('\ntester results')
for tester in test_matrix.keys():
print(f'\n{tester}:')
+ print(' expected results:')
+ for test in collect_expected_results(test_matrix[tester]['expected_results']):
+ print(f' {test.name}: {test.result}')
+ print(' actual results:')
for test in collect_actual_results(results[tester]):
- print(f'{test.name}: {test.result}')
+ if test.name in test_matrix[tester]['expected_results']:
+ print(f' {test.name}: {test.result} (exp: {test_matrix[tester]["expected_results"][test.name]})')
+ else:
+ print(f' {test.name}: {test.result}')
assert True