diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5b1febd30..e8a5d165e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -87,3 +87,19 @@ jobs: - name: Run pylint shell: bash {0} run: testing/pylint/test_pylint + + testrun_package: + name: Package + runs-on: ubuntu-22.04 + timeout-minutes: 5 + steps: + - name: Checkout source + uses: actions/checkout@v2.3.4 + - name: Package Testrun + shell: bash {0} + run: cmd/package + - name: Archive package + uses: actions/upload-artifact@v3 + with: + name: Testrun Installer + path: testrun*.deb \ No newline at end of file diff --git a/README.md b/README.md index 5ed2d03de..6542c3bce 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ Testrun cannot automate everything, and so additional manual testing may be requ - 2x USB ethernet adapter (One may be built in ethernet) - Internet connection ### Software - - Python 3 (Already available on Ubuntu LTS) - - Docker - [Install guide](https://docs.docker.com/engine/install/ubuntu/) - - Open vSwitch ``sudo apt-get install openvswitch-common openvswitch-switch`` +- Python3 libraries: ``sudo apt-get install python3-dev python3-venv`` +- Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) +- System dependencies: ``sudo apt-get install openvswitch-common openvswitch-switch build-essential net-tools`` ## Get started ▶️ Once you have met the hardware and software requirements, you can get started with Testrun by following the [Get started guide](docs/get_started.md). diff --git a/bin/testrun b/bin/testrun index 82c8ab237..41af70ffe 100755 --- a/bin/testrun +++ b/bin/testrun @@ -40,6 +40,6 @@ source venv/bin/activate # Set the PYTHONPATH to include the "src" directory export PYTHONPATH="$TESTRUNPATH/framework/python/src" -python -u framework/python/src/core/test_runner.py $@ +python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log deactivate \ No newline at end of file diff --git a/cmd/build b/cmd/build index 7e69393c8..5143e8902 100755 --- a/cmd/build +++ b/cmd/build @@ -20,7 +20,7 @@ echo Building docker images # Build user interface echo Building user interface mkdir -p build/ui -docker build -t test-run/ui -f modules/ui/ui.Dockerfile . > build/ui/ui.log 2>&1 +docker build -t test-run/ui -f modules/ui/ui.Dockerfile . 2>&1 | tee build/ui/ui.log # Build network modules echo Building network modules @@ -28,7 +28,7 @@ mkdir -p build/network for dir in modules/network/* ; do module=$(basename $dir) echo Building network module $module... - docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . > build/network/$module.log 2>&1 + docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/network/$module.log done # Build validators @@ -37,7 +37,7 @@ mkdir -p build/devices for dir in modules/devices/* ; do module=$(basename $dir) echo Building validator module $module... - docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . > build/devices/$module.log 2>&1 + docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/devices/$module.log done # Build test modules @@ -46,7 +46,7 @@ mkdir -p build/test for dir in modules/test/* ; do module=$(basename $dir) echo Building test module $module... - docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . > build/test/$module.log 2>&1 + docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . 2>&1 | tee build/test/$module.log done echo Finished building modules \ No newline at end of file diff --git a/cmd/install b/cmd/install index 929f9136c..0b6ac92de 100755 --- a/cmd/install +++ b/cmd/install @@ -26,7 +26,7 @@ source venv/bin/activate pip3 install -r framework/requirements.txt # Copy the default configuration -cp -u local/system.json.example local/system.json +cp -n local/system.json.example local/system.json deactivate diff --git a/docs/get_started.md b/docs/get_started.md index fcdd21fe7..31f48181c 100644 --- a/docs/get_started.md +++ b/docs/get_started.md @@ -13,16 +13,20 @@ Before starting with Testrun, ensure you have the following hardware: ### Software Ensure the following software is installed on your Ubuntu LTS PC: - -- Python 3 (already available on Ubuntu LTS) -- Docker - Installation Guide: [https://docs.docker.com/engine/install/](https://docs.docker.com/engine/install/) -- Open vSwitch ``sudo apt-get install openvswitch-common openvswitch-switch`` +- Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) +- System dependencies (These will be installed automatically when installing Testrun if not already installed): + - Python3-dev + - Python3-venv + - Openvswitch Common + - Openvswitch Switch + - Build Essential + - Net Tools ## Installation 1. Download the latest version of Testrun from the [releases page](https://github.com/google/test-run/releases) -2. Install the package using ``sudo dpkg -i testrun_*.deb`` +2. Install the package using ``sudo apt install ./testrun*.deb`` ## Test Your Device @@ -31,14 +35,18 @@ Ensure the following software is installed on your Ubuntu LTS PC: - Connect one USB Ethernet adapter to the internet source (e.g., router or switch) using an Ethernet cable. - Connect the other USB Ethernet adapter directly to the IoT device you want to test using an Ethernet cable. + **NOTE: Both adapters should be disabled in the host system (IPv4, IPv6 and general). You can do this by going to Settings > Network** + 2. Start Testrun. -Start Testrun with the command `sudo testrun` +Start Testrun with the command `sudo testrun --no-validate` - To run Testrun in network-only mode (without running any tests), use the `--net-only` option. - To skip network validation before use and not launch the faux device on startup, use the `--no-validate` option. + - To run Testrun with just one interface (connected to the device), use the ``--single-intf`` option. + # Troubleshooting If you encounter any issues or need assistance, consider the following: @@ -46,4 +54,4 @@ If you encounter any issues or need assistance, consider the following: - Ensure that all hardware and software prerequisites are met. - Verify that the network interfaces are connected correctly. - Check the configuration settings. -- Refer to the Test Run documentation or ask for further assistance from the support team. +- Refer to the Testrun documentation or ask for assistance in the issues page: https://github.com/google/testrun/issues diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index ef8448bf8..139bd2dd0 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -129,8 +129,11 @@ async def start_test_run(self, request: Request, response: Response): device = self._session.get_device(body_json["device"]["mac_addr"]) # Check Test Run is not already running - if self._test_run.get_session().get_status() != "Idle": - LOGGER.debug("Test Run is already running. Cannot start another instance") + if self._test_run.get_session().get_status() in [ + "In Progress", + "Waiting for Device", + ]: + LOGGER.debug("Testrun is already running. Cannot start another instance") response.status_code = status.HTTP_409_CONFLICT return self._generate_msg(False, "Test Run is already running") diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py index af05f2a2f..da26e0700 100644 --- a/framework/python/src/common/testreport.py +++ b/framework/python/src/common/testreport.py @@ -14,14 +14,16 @@ """Store previous test run information.""" +import os from datetime import datetime from weasyprint import HTML from io import BytesIO DATE_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' +DEVICES_DIR = '/usr/local/testrun/local/devices' class TestReport(): - """Represents a previous Test Run report.""" + """Represents a previous Testrun report.""" def __init__(self, status='Non-Compliant', @@ -35,6 +37,7 @@ def __init__(self, self._finished = finished self._total_tests = total_tests self._results = [] + self._report = '' def get_status(self): return self._status @@ -55,6 +58,9 @@ def get_duration(self): def add_test(self, test): self._results.append(test) + def get_report_url(self): + return self._report + def to_json(self): report_json = {} report_json['device'] = self._device @@ -63,6 +69,7 @@ def to_json(self): report_json['finished'] = self._finished.strftime(DATE_TIME_FORMAT) report_json['tests'] = {'total': self._total_tests, 'results': self._results} + report_json['report'] = self._report return report_json def from_json(self, json_file): @@ -71,7 +78,7 @@ def from_json(self, json_file): self._device['manufacturer'] = json_file['device']['manufacturer'] self._device['model'] = json_file['device']['model'] - if 'firmware' in self._device: + if 'firmware' in json_file['device']: self._device['firmware'] = json_file['device']['firmware'] self._status = json_file['status'] @@ -79,6 +86,9 @@ def from_json(self, json_file): self._finished = datetime.strptime(json_file['finished'], DATE_TIME_FORMAT) self._total_tests = json_file['tests']['total'] + if 'report' in json_file: + self._report = json_file['report'] + # Loop through test results for test_result in json_file['tests']['results']: self.add_test(test_result) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index f67cbb835..e1b3f9aab 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -374,13 +374,14 @@ def _device_discovered(self, mac_addr): self.get_session().set_target_device(device) + self._set_status('In Progress') + LOGGER.info( f'Discovered {device.manufacturer} {device.model} on the network. ' + 'Waiting for device to obtain IP') def _device_stable(self, mac_addr): LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') - self._set_status('In Progress') result = self._test_orc.run_test_modules() self._set_status(result) @@ -392,6 +393,8 @@ def _set_status(self, status): def start_ui(self): + self._stop_ui() + LOGGER.info('Starting UI') client = docker.from_env() diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 975cde112..cc841f0ca 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -448,7 +448,7 @@ def _get_network_module(self, name): def _start_network_service(self, net_module): - LOGGER.debug('Starting net service ' + net_module.display_name) + LOGGER.debug('Starting network service ' + net_module.display_name) network = 'host' if net_module.net_config.host else PRIVATE_DOCKER_NET LOGGER.debug(f"""Network: {network}, image name: {net_module.image_name}, container name: {net_module.container_name}""") @@ -473,7 +473,7 @@ def _start_network_service(self, net_module): self._attach_service_to_network(net_module) def _stop_service_module(self, net_module, kill=False): - LOGGER.debug('Stopping Service container ' + net_module.container_name) + LOGGER.debug('Stopping network container ' + net_module.container_name) try: container = self._get_service_container(net_module) if container is not None: diff --git a/framework/python/src/test_orc/module.py b/framework/python/src/test_orc/module.py index 6f3c544a1..0cf0286d7 100644 --- a/framework/python/src/test_orc/module.py +++ b/framework/python/src/test_orc/module.py @@ -24,6 +24,7 @@ class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-at name: str = None display_name: str = None description: str = None + enabled: bool = True tests: list = field(default_factory=lambda: []) # Docker settings diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 1f0a585cb..d8ae4b5f2 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -33,6 +33,7 @@ LOG_REGEX = r"^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_" SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports" DEVICE_ROOT_CERTS = "local/root_certs" +TESTRUN_DIR = "/usr/local/testrun" class TestOrchestrator: @@ -82,8 +83,10 @@ def run_test_modules(self): LOGGER.info("All tests complete") self._session.stop() + report = TestReport().from_json(self._generate_report()) device.add_report(report) + self._write_reports(report) self._test_in_progress = False self._timestamp_results(device) @@ -97,6 +100,7 @@ def run_test_modules(self): return report.get_status() def _write_reports(self, test_report): + out_dir = os.path.join( self._root_path, RUNTIME_DIR, self._session.get_target_device().mac_addr.replace(":", "")) @@ -118,21 +122,35 @@ def _write_reports(self, test_report): def _generate_report(self): report = {} - report["device"] = self._session.get_target_device().to_dict() - report["started"] = self._session.get_started().strftime( + report["device"] = self.get_session().get_target_device().to_dict() + report["started"] = self.get_session().get_started().strftime( "%Y-%m-%d %H:%M:%S") - report["finished"] = self._session.get_finished().strftime( + report["finished"] = self.get_session().get_finished().strftime( "%Y-%m-%d %H:%M:%S") report["status"] = self._calculate_result() - report["tests"] = self._session.get_report_tests() + report["tests"] = self.get_session().get_report_tests() + report["report"] = "file://" + os.path.join( + TESTRUN_DIR, + SAVED_DEVICE_REPORTS.replace( + "{device_folder}", + self.get_session().get_target_device().device_folder), + self.get_session().get_finished().strftime( + "%Y-%m-%dT%H:%M:%S"), + "report.pdf" + ) + out_file = os.path.join( self._root_path, RUNTIME_DIR, self._session.get_target_device().mac_addr.replace(":", ""), "report.json") + LOGGER.debug(f"Saving report to {out_file}") + + # Write report to runtime directory with open(out_file, "w", encoding="utf-8") as f: json.dump(report, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") + return report def _calculate_result(self): @@ -197,7 +215,7 @@ def _timestamp_results(self, device): device.mac_addr.replace(":", "")) # Define the destination results directory with timestamp - cur_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + cur_time = self.get_session().get_finished().strftime("%Y-%m-%dT%H:%M:%S") completed_results_dir = os.path.join( SAVED_DEVICE_REPORTS.replace("{device_folder}", device.device_folder), cur_time) @@ -225,7 +243,7 @@ def _run_test_module(self, module): device = self._session.get_target_device() - if module is None or not module.enable_container: + if module is None or not module.enable_container or not module.enabled: return if not self._is_module_enabled(module, device): @@ -388,6 +406,10 @@ def _load_test_module(self, module_dir): module.name = module_json["config"]["meta"]["name"] module.display_name = module_json["config"]["meta"]["display_name"] module.description = module_json["config"]["meta"]["description"] + + if "enabled" in module_json["config"]: + module.enabled = module_json["config"]["enabled"] + module.dir = os.path.join(self._path, modules_dir, module_dir) module.dir_name = module_dir module.build_file = module_dir + ".Dockerfile" @@ -495,3 +517,6 @@ def get_test_case(self, name): if test_case.name == name: return test_case return None + + def get_session(self): + return self._session diff --git a/make/.gitignore b/make/.gitignore index 1be953b79..d93bca69e 100644 --- a/make/.gitignore +++ b/make/.gitignore @@ -1,2 +1,3 @@ usr/ -bin/ \ No newline at end of file +bin/ +DEBIAN/postinst \ No newline at end of file diff --git a/make/DEBIAN/control b/make/DEBIAN/control index 20463e996..9ad0ed2de 100644 --- a/make/DEBIAN/control +++ b/make/DEBIAN/control @@ -2,5 +2,7 @@ Package: Testrun Version: 1.0 Architecture: amd64 Maintainer: Google +Homepage: https://github.com/google/testrun +Bugs: https://github.com/google/testrun/issues Description: Automatically verify IoT device network behavior -Depends: libpangocairo-1.0-0, openvswitch-common, openvswitch-switch, python3 +Depends: libpangocairo-1.0-0, openvswitch-common, openvswitch-switch, build-essential, python3, python3-dev, python3-venv, net-tools diff --git a/make/DEBIAN/postinst b/make/DEBIAN/postinst index 929f9136c..0b6ac92de 100755 --- a/make/DEBIAN/postinst +++ b/make/DEBIAN/postinst @@ -26,7 +26,7 @@ source venv/bin/activate pip3 install -r framework/requirements.txt # Copy the default configuration -cp -u local/system.json.example local/system.json +cp -n local/system.json.example local/system.json deactivate diff --git a/modules/test/baseline/conf/module_config.json b/modules/test/baseline/conf/module_config.json index ac1148fd2..cc78ce0a0 100644 --- a/modules/test/baseline/conf/module_config.json +++ b/modules/test/baseline/conf/module_config.json @@ -22,7 +22,7 @@ "name": "baseline.non_compliant", "test_description": "Simulate a non-compliant test", "expected_behavior": "A non-compliant test result is generated", - "required_result": "Required" + "required_result": "Recommended" }, { "name": "baseline.informational", diff --git a/modules/test/conn/conf/module_config.json b/modules/test/conn/conf/module_config.json index 4cea03c9d..5253c59f9 100644 --- a/modules/test/conn/conf/module_config.json +++ b/modules/test/conn/conf/module_config.json @@ -67,26 +67,6 @@ ] } }, - { - "name": "connection.private_address", - "test_description": "The device under test accepts an IP address that is compliant with RFC 1918 Address Allocation for Private Internets.", - "expected_behavior": "The device under test accepts IP addresses within all ranges specified in RFC 1918 and communicates using these addresses. The Internet Assigned Numbers Authority (IANA) has reserved the following three blocks of the IP address space for private internets. 10.0.0.0 - 10.255.255.255.255 (10/8 prefix). 172.16.0.0 - 172.31.255.255 (172.16/12 prefix). 192.168.0.0 - 192.168.255.255 (192.168/16 prefix)", - "required_result": "Required", - "config": [ - { - "start": "10.0.0.100", - "end": "10.0.0.200" - }, - { - "start":"172.16.0.0", - "end":"172.16.255.255" - }, - { - "start":"192.168.0.0", - "end":"192.168.255.255" - } - ] - }, { "name": "connection.single_ip", "test_description": "The network switch port connected to the device reports only one IP address for the device under test.", diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index eacd54f94..8bbee5757 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -146,7 +146,11 @@ def _connection_single_ip(self): for mac_address in mac_addresses: LOGGER.debug('DHCPREQUEST from MAC address: ' + mac_address) result &= self._device_mac.upper() == mac_address - return result + + if result: + return result, 'Device is using a single IP address' + else: + return result, 'Device is using multiple IP addresses' def _connection_target_ping(self): LOGGER.info('Running connection.target_ping') @@ -157,9 +161,12 @@ def _connection_target_ping(self): if self._device_ipv4_addr is None: LOGGER.error('No device IP could be resolved') - sys.exit(1) + return False, 'Could not resolve device IP' else: - return self._ping(self._device_ipv4_addr) + if self._ping(self._device_ipv4_addr): + return True, 'Device responds to ping' + else: + return False, 'Device does not respond to ping' def _connection_ipaddr_ip_change(self): result = None diff --git a/modules/test/protocol/conf/module_config.json b/modules/test/protocol/conf/module_config.json index e25603b86..0fa83afb8 100644 --- a/modules/test/protocol/conf/module_config.json +++ b/modules/test/protocol/conf/module_config.json @@ -1,5 +1,6 @@ { "config": { + "enabled": false, "meta": { "name": "protocol", "display_name": "Protocol", diff --git a/testing/api/test_api b/testing/api/test_api index d4269d979..4b379c717 100755 --- a/testing/api/test_api +++ b/testing/api/test_api @@ -25,12 +25,11 @@ sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils # Setup device network sudo ip link add dev dummynet type dummy sudo ip link add dev endev0a type veth peer name endev0b +sudo ip link set dev dummynet up sudo ip link set dev endev0a up sudo ip link set dev endev0b up sudo docker network create -d macvlan -o parent=endev0b endev0 -sudo ip link add dev dummynet type dummy - # Start OVS sudo /usr/share/openvswitch/scripts/ovs-ctl start @@ -40,9 +39,14 @@ sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f ./test sudo chown -R $USER local # Copy configuration to testrun -sudo cp testing/baseline/system.json $TESTRUN_DIR/local/system.json +sudo cp testing/api/system.json $TESTRUN_DIR/local/system.json # Needs to be sudo because this invokes bin/testrun sudo $TESTRUN_DIR/venv/bin/python3 -m pytest -v testing/api/test_api.py +# Clean up network interfaces after use +sudo docker network rm endev0 +sudo ip link del dev endev0a +sudo ip link del dev dummynet + exit $? \ No newline at end of file diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 4a4080fe3..c56ef3d73 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -390,7 +390,7 @@ def test_trigger_run(testing_devices, testrun): until_true( lambda: query_system_status().lower() == "compliant", "system status is `complete`", - 900, + 600, ) stop_test_device("x123") @@ -473,8 +473,6 @@ def test_stop_running_not_running(testrun): assert False -# TODO enable test because functionality is broken -@pytest.mark.skip() def test_multiple_runs(testing_devices, testrun): payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} r = requests.post(f"{API}/system/start", data=json.dumps(payload)) @@ -534,7 +532,8 @@ def test_multiple_runs(testing_devices, testrun): @pytest.mark.skip() def test_create_invalid_chars(empty_devices_dir, testrun): # local_delete_devices(ALL_DEVICES) - # We must start test run with no devices in local/devices for this test to function as expected! + # We must start test run with no devices in local/devices for this test + # to function as expected! assert len(local_get_devices()) == 0 # Test adding device