bug fixes

.github/workflows/testing.yml

@@ -87,3 +87,19 @@ jobs:
       - name: Run pylint
         shell: bash {0}
         run: testing/pylint/test_pylint
+
+  testrun_package:
+    name: Package
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.4
+      - name: Package Testrun
+        shell: bash {0}
+        run: cmd/package
+      - name: Archive package
+        uses: actions/upload-artifact@v3
+        with:
+          name: Testrun Installer
+          path: testrun*.deb

README.md

@@ -30,9 +30,9 @@ Testrun cannot automate everything, and so additional manual testing may be requ
  - 2x USB ethernet adapter (One may be built in ethernet)
  - Internet connection
 ### Software
- - Python 3 (Already available on Ubuntu LTS)
- - Docker - [Install guide](https://docs.docker.com/engine/install/ubuntu/)
- - Open vSwitch ``sudo apt-get install openvswitch-common openvswitch-switch``
+- Python3 libraries: ``sudo apt-get install python3-dev python3-venv``
+- Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository)
+- System dependencies: ``sudo apt-get install openvswitch-common openvswitch-switch build-essential net-tools``
 
 ## Get started ▶️
 Once you have met the hardware and software requirements, you can get started with Testrun by following the [Get started guide](docs/get_started.md).

bin/testrun

@@ -40,6 +40,6 @@ source venv/bin/activate
 
 # Set the PYTHONPATH to include the "src" directory
 export PYTHONPATH="$TESTRUNPATH/framework/python/src"
-python -u framework/python/src/core/test_runner.py $@
+python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log
 
 deactivate

cmd/build

@@ -20,15 +20,15 @@ echo Building docker images
 # Build user interface
 echo Building user interface
 mkdir -p build/ui
-docker build -t test-run/ui -f modules/ui/ui.Dockerfile . > build/ui/ui.log 2>&1
+docker build -t test-run/ui -f modules/ui/ui.Dockerfile . 2>&1 | tee build/ui/ui.log
 
 # Build network modules
 echo Building network modules
 mkdir -p build/network
 for dir in modules/network/* ; do
   module=$(basename $dir)
   echo Building network module $module...
-  docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . > build/network/$module.log 2>&1
+  docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/network/$module.log
 done
 
 # Build validators
@@ -37,7 +37,7 @@ mkdir -p build/devices
 for dir in modules/devices/* ; do
   module=$(basename $dir)
   echo Building validator module $module...
-  docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . > build/devices/$module.log 2>&1
+  docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/devices/$module.log
 done
 
 # Build test modules
@@ -46,7 +46,7 @@ mkdir -p build/test
 for dir in modules/test/* ; do
   module=$(basename $dir)
   echo Building test module $module...
-  docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . > build/test/$module.log 2>&1
+  docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . 2>&1 | tee build/test/$module.log
 done
 
 echo Finished building modules

cmd/install

@@ -26,7 +26,7 @@ source venv/bin/activate
 pip3 install -r framework/requirements.txt
 
 # Copy the default configuration
-cp -u local/system.json.example local/system.json
+cp -n local/system.json.example local/system.json
 
 deactivate
 

docs/get_started.md

@@ -13,16 +13,20 @@ Before starting with Testrun, ensure you have the following hardware:
 ### Software
 
 Ensure the following software is installed on your Ubuntu LTS PC:
-
-- Python 3 (already available on Ubuntu LTS)
-- Docker - Installation Guide: [https://docs.docker.com/engine/install/](https://docs.docker.com/engine/install/)
-- Open vSwitch ``sudo apt-get install openvswitch-common openvswitch-switch``
+- Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository)
+- System dependencies (These will be installed automatically when installing Testrun if not already installed): 
+   - Python3-dev
+   - Python3-venv
+   - Openvswitch Common
+   - Openvswitch Switch
+   - Build Essential
+   - Net Tools
 
 ## Installation
 
 1. Download the latest version of Testrun from the [releases page](https://github.com/google/test-run/releases)
 
-2. Install the package using ``sudo dpkg -i testrun_*.deb``
+2. Install the package using ``sudo apt install ./testrun*.deb``
 
 ## Test Your Device
 
@@ -31,19 +35,23 @@ Ensure the following software is installed on your Ubuntu LTS PC:
    - Connect one USB Ethernet adapter to the internet source (e.g., router or switch) using an Ethernet cable.
    - Connect the other USB Ethernet adapter directly to the IoT device you want to test using an Ethernet cable.
 
+   **NOTE: Both adapters should be disabled in the host system (IPv4, IPv6 and general). You can do this by going to Settings > Network** 
+
 2. Start Testrun.
 
-Start Testrun with the command `sudo testrun`
+Start Testrun with the command `sudo testrun --no-validate`
 
    - To run Testrun in network-only mode (without running any tests), use the `--net-only` option.
 
    - To skip network validation before use and not launch the faux device on startup, use the `--no-validate` option.
 
+   - To run Testrun with just one interface (connected to the device), use the ``--single-intf`` option.
+
 # Troubleshooting
 
 If you encounter any issues or need assistance, consider the following:
 
 - Ensure that all hardware and software prerequisites are met.
 - Verify that the network interfaces are connected correctly.
 - Check the configuration settings.
-- Refer to the Test Run documentation or ask for further assistance from the support team.
+- Refer to the Testrun documentation or ask for assistance in the issues page: https://github.com/google/testrun/issues

framework/python/src/api/api.py

@@ -129,8 +129,11 @@ async def start_test_run(self, request: Request, response: Response):
     device = self._session.get_device(body_json["device"]["mac_addr"])
 
     # Check Test Run is not already running
-    if self._test_run.get_session().get_status() != "Idle":
-      LOGGER.debug("Test Run is already running. Cannot start another instance")
+    if self._test_run.get_session().get_status() in [
+        "In Progress",
+        "Waiting for Device",
+      ]:
+      LOGGER.debug("Testrun is already running. Cannot start another instance")
       response.status_code = status.HTTP_409_CONFLICT
       return self._generate_msg(False, "Test Run is already running")
 

framework/python/src/common/testreport.py

@@ -14,14 +14,16 @@
 
 """Store previous test run information."""
 
+import os
 from datetime import datetime
 from weasyprint import HTML
 from io import BytesIO
 
 DATE_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
+DEVICES_DIR = '/usr/local/testrun/local/devices'
 
 class TestReport():
-  """Represents a previous Test Run report."""
+  """Represents a previous Testrun report."""
 
   def __init__(self,
                status='Non-Compliant',
@@ -35,6 +37,7 @@ def __init__(self,
     self._finished = finished
     self._total_tests = total_tests
     self._results = []
+    self._report = ''
 
   def get_status(self):
     return self._status
@@ -55,6 +58,9 @@ def get_duration(self):
   def add_test(self, test):
     self._results.append(test)
 
+  def get_report_url(self):
+    return self._report
+
   def to_json(self):
     report_json = {}
     report_json['device'] = self._device
@@ -63,6 +69,7 @@ def to_json(self):
     report_json['finished'] = self._finished.strftime(DATE_TIME_FORMAT)
     report_json['tests'] = {'total': self._total_tests,
                             'results': self._results}
+    report_json['report'] = self._report
     return report_json
 
   def from_json(self, json_file):
@@ -71,14 +78,17 @@ def from_json(self, json_file):
     self._device['manufacturer'] = json_file['device']['manufacturer']
     self._device['model'] = json_file['device']['model']
 
-    if 'firmware' in self._device:
+    if 'firmware' in json_file['device']:
       self._device['firmware'] = json_file['device']['firmware']
 
     self._status = json_file['status']
     self._started = datetime.strptime(json_file['started'], DATE_TIME_FORMAT)
     self._finished = datetime.strptime(json_file['finished'], DATE_TIME_FORMAT)
     self._total_tests = json_file['tests']['total']
 
+    if 'report' in json_file:
+      self._report = json_file['report']
+
     # Loop through test results
     for test_result in json_file['tests']['results']:
       self.add_test(test_result)

framework/python/src/core/testrun.py

@@ -374,13 +374,14 @@ def _device_discovered(self, mac_addr):
 
       self.get_session().set_target_device(device)
 
+    self._set_status('In Progress')
+
     LOGGER.info(
         f'Discovered {device.manufacturer} {device.model} on the network. ' +
         'Waiting for device to obtain IP')
 
   def _device_stable(self, mac_addr):
     LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.')
-    self._set_status('In Progress')
     result = self._test_orc.run_test_modules()
     self._set_status(result)
 
@@ -392,6 +393,8 @@ def _set_status(self, status):
 
   def start_ui(self):
 
+    self._stop_ui()
+
     LOGGER.info('Starting UI')
 
     client = docker.from_env()

framework/python/src/net_orc/network_orchestrator.py

@@ -448,7 +448,7 @@ def _get_network_module(self, name):
 
   def _start_network_service(self, net_module):
 
-    LOGGER.debug('Starting net service ' + net_module.display_name)
+    LOGGER.debug('Starting network service ' + net_module.display_name)
     network = 'host' if net_module.net_config.host else PRIVATE_DOCKER_NET
     LOGGER.debug(f"""Network: {network}, image name: {net_module.image_name},
                      container name: {net_module.container_name}""")
@@ -473,7 +473,7 @@ def _start_network_service(self, net_module):
       self._attach_service_to_network(net_module)
 
   def _stop_service_module(self, net_module, kill=False):
-    LOGGER.debug('Stopping Service container ' + net_module.container_name)
+    LOGGER.debug('Stopping network container ' + net_module.container_name)
     try:
       container = self._get_service_container(net_module)
       if container is not None:

framework/python/src/test_orc/module.py

@@ -24,6 +24,7 @@ class TestModule:  # pylint: disable=too-few-public-methods,too-many-instance-at
   name: str = None
   display_name: str = None
   description: str = None
+  enabled: bool = True
   tests: list = field(default_factory=lambda: [])
 
   # Docker settings

framework/python/src/test_orc/test_orchestrator.py

@@ -33,6 +33,7 @@
 LOG_REGEX = r"^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_"
 SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports"
 DEVICE_ROOT_CERTS = "local/root_certs"
+TESTRUN_DIR = "/usr/local/testrun"
 
 
 class TestOrchestrator:
@@ -82,8 +83,10 @@ def run_test_modules(self):
     LOGGER.info("All tests complete")
 
     self._session.stop()
+
     report = TestReport().from_json(self._generate_report())
     device.add_report(report)
+
     self._write_reports(report)
     self._test_in_progress = False
     self._timestamp_results(device)
@@ -97,6 +100,7 @@ def run_test_modules(self):
     return report.get_status()
 
   def _write_reports(self, test_report):
+
     out_dir = os.path.join(
         self._root_path, RUNTIME_DIR,
         self._session.get_target_device().mac_addr.replace(":", ""))
@@ -118,21 +122,35 @@ def _write_reports(self, test_report):
   def _generate_report(self):
 
     report = {}
-    report["device"] = self._session.get_target_device().to_dict()
-    report["started"] = self._session.get_started().strftime(
+    report["device"] = self.get_session().get_target_device().to_dict()
+    report["started"] = self.get_session().get_started().strftime(
         "%Y-%m-%d %H:%M:%S")
-    report["finished"] = self._session.get_finished().strftime(
+    report["finished"] = self.get_session().get_finished().strftime(
         "%Y-%m-%d %H:%M:%S")
     report["status"] = self._calculate_result()
-    report["tests"] = self._session.get_report_tests()
+    report["tests"] = self.get_session().get_report_tests()
+    report["report"] = "file://" + os.path.join(
+      TESTRUN_DIR,
+      SAVED_DEVICE_REPORTS.replace(
+        "{device_folder}",
+        self.get_session().get_target_device().device_folder),
+        self.get_session().get_finished().strftime(
+          "%Y-%m-%dT%H:%M:%S"),
+      "report.pdf"
+    )
+
     out_file = os.path.join(
         self._root_path, RUNTIME_DIR,
         self._session.get_target_device().mac_addr.replace(":", ""),
         "report.json")
 
+    LOGGER.debug(f"Saving report to {out_file}")
+
+    # Write report to runtime directory
     with open(out_file, "w", encoding="utf-8") as f:
       json.dump(report, f, indent=2)
     util.run_command(f"chown -R {self._host_user} {out_file}")
+
     return report
 
   def _calculate_result(self):
@@ -197,7 +215,7 @@ def _timestamp_results(self, device):
                                    device.mac_addr.replace(":", ""))
 
     # Define the destination results directory with timestamp
-    cur_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+    cur_time = self.get_session().get_finished().strftime("%Y-%m-%dT%H:%M:%S")
     completed_results_dir = os.path.join(
         SAVED_DEVICE_REPORTS.replace("{device_folder}", device.device_folder),
         cur_time)
@@ -225,7 +243,7 @@ def _run_test_module(self, module):
 
     device = self._session.get_target_device()
 
-    if module is None or not module.enable_container:
+    if module is None or not module.enable_container or not module.enabled:
       return
 
     if not self._is_module_enabled(module, device):
@@ -388,6 +406,10 @@ def _load_test_module(self, module_dir):
     module.name = module_json["config"]["meta"]["name"]
     module.display_name = module_json["config"]["meta"]["display_name"]
     module.description = module_json["config"]["meta"]["description"]
+
+    if "enabled" in module_json["config"]:
+      module.enabled = module_json["config"]["enabled"]
+
     module.dir = os.path.join(self._path, modules_dir, module_dir)
     module.dir_name = module_dir
     module.build_file = module_dir + ".Dockerfile"
@@ -495,3 +517,6 @@ def get_test_case(self, name):
       if test_case.name == name:
         return test_case
     return None
+
+  def get_session(self):
+    return self._session

make/.gitignore

@@ -1,2 +1,3 @@
 usr/
-bin/
+bin/
+DEBIAN/postinst

make/DEBIAN/control

@@ -2,5 +2,7 @@ Package: Testrun
 Version: 1.0
 Architecture: amd64
 Maintainer: Google <boddey@google.com>
+Homepage: https://github.com/google/testrun
+Bugs: https://github.com/google/testrun/issues
 Description: Automatically verify IoT device network behavior
-Depends: libpangocairo-1.0-0, openvswitch-common, openvswitch-switch, python3
+Depends: libpangocairo-1.0-0, openvswitch-common, openvswitch-switch, build-essential, python3, python3-dev, python3-venv, net-tools

make/DEBIAN/postinst

@@ -26,7 +26,7 @@ source venv/bin/activate
 pip3 install -r framework/requirements.txt
 
 # Copy the default configuration
-cp -u local/system.json.example local/system.json
+cp -n local/system.json.example local/system.json
 
 deactivate