Improve dev process

.github/workflows/testing.yml

@@ -17,13 +17,10 @@ jobs:
       - name: Install dependencies
         shell: bash {0}
         run: cmd/prepare
-      - name: Package Testrun
-        shell: bash {0}
-        run: cmd/package
       - name: Install Testrun
         shell: bash {0}
         timeout-minutes: 10
-        run: sudo dpkg -i testrun*.deb
+        run: TESTRUN_DIR=. cmd/install
       - name: Run baseline tests
         shell: bash {0}
         run: testing/baseline/test_baseline
@@ -71,19 +68,16 @@ jobs:
       - name: Install dependencies
         shell: bash {0}
         run: cmd/prepare
-      - name: Package Testrun  
-        shell: bash {0}
-        run: cmd/package
       - name: Install Testrun
         shell: bash {0}
-        run: sudo dpkg -i testrun*.deb
+        run: TESTRUN_DIR=. cmd/install
         timeout-minutes: 30
       - name: Run tests
         shell: bash {0}
         run: testing/api/test_api
       - name: Archive runtime results
         if: ${{ always() }}
-        run: sudo tar --exclude-vcs -czf runtime.tgz /usr/local/testrun/runtime/ /usr/local/testrun/local/
+        run: sudo tar --exclude-vcs -czf runtime.tgz runtime/ local/
       - name: Upload runtime results
         uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
         if: ${{ always() }}

.gitignore

@@ -15,4 +15,6 @@ testing/unit/tls/output/
 testing/unit/report/output/
 
 *.deb
-make/DEBIAN/postinst
+make/DEBIAN/postinst
+
+testrun.log

README.md

@@ -51,25 +51,30 @@ If the application has come across a problem at any point during setup or use, p
 The contributing requirements can be found in [CONTRIBUTING.md](CONTRIBUTING.md). In short, checkout the [Google CLA](https://cla.developers.google.com/) site to get started.
 
 ## FAQ :raising_hand:
-1) What device networking functionality is validated by Testrun?
+1) I have an issue whilst installing/upgrading Testrun, what do I do?
+
+  Sometimes, issues may arise when installing or upgrading Testrun - this may happen due to one of many reasons due to the nature of the application. However, most of the time, it can be resolved by following a full Testrun re-install by using these commands:
+   - ```sudo docker system prune -a```
+   - ```sudo apt install ./testrun-*.deb```
+
+2) What device networking functionality is validated by Testrun?
 
   Best practices and requirements for IoT devices are constantly changing due to technological advances and discovery of vulnerabilities. 
   The current expectations for IoT devices on Google deployments can be found in the [Application Security Requirements for IoT Devices](https://partner-security.withgoogle.com/docs/iot_requirements).
   Testrun aims to automate as much of the Application Security Requirements as possible.
 
-2) What services are provided on the virtual network?
+3) What services are provided on the virtual network?
 
   The following are network services that are containerized and accessible to the device under test though are likely to change over time:
  - DHCP in failover configuration with internet connectivity
  - IPv6 SLAAC
  - DNS
  - NTPv4
- - 802.1x Port Based Authentication
 
-3) Can I run Testrun on a virtual machine?
+4) Can I run Testrun on a virtual machine?
 
   Testrun can be virtualized if the 2x ethernet adapters are passed through to a Virtual Box VM as a USB device rather than managed network adapters. A full guide will be provided once virtualization of Testrun has been fully tested.
 
- 4) Can I connect multiple devices to Testrun?
+ 5) Can I connect multiple devices to Testrun?
 
   In short, Yes you can. The way in which multiple devices could be tested simultaneously is yet to be decided. However, if you simply want to add field/peer devices during runtime (even another laptop performing manual testing) then you may connect the USB ethernet adapter to an unmanaged switch.

bin/testrun

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Check that user is root
 if [[ "$EUID" -ne 0 ]]; then
     echo "Must run as root. Use sudo $0"
     exit 1
@@ -22,11 +23,17 @@ fi
 # Ensure that /var/run/netns folder exists
 sudo mkdir -p /var/run/netns
 
-export TESTRUNPATH=/usr/local/testrun
-cd $TESTRUNPATH
+# Get the directory of the running script
+FILE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-# Create device folder if it doesn't exist
-mkdir -p local/devices
+# Check if Testrun was installed as a package
+if [ $FILE_PATH == "/usr/bin" ] ; then
+  export TESTRUN_PATH="/usr/local/testrun"
+else
+  export TESTRUN_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd)
+fi
+
+cd $TESTRUN_PATH
 
 # Remove existing runtime data
 rm -rf runtime/*
@@ -35,7 +42,7 @@ rm -rf runtime/*
 source venv/bin/activate
 
 # Set the PYTHONPATH to include the "src" directory
-export PYTHONPATH="$TESTRUNPATH/framework/python/src"
+export PYTHONPATH="$TESTRUN_PATH/framework/python/src"
 python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log
 
 deactivate

cmd/build

@@ -38,16 +38,25 @@ echo Building docker images
 
 # Build user interface
 echo Building user interface
-mkdir -p build/ui
-docker build -t test-run/ui -f modules/ui/ui.Dockerfile . 2>&1 | tee build/ui/ui.log
+if docker build -t test-run/ui -f modules/ui/ui.Dockerfile . ; then
+  echo Successully built the user interface
+else
+  echo An error occured whilst building the user interface
+  exit 1
+fi
 
 # Build network modules
 echo Building network modules
 mkdir -p build/network
 for dir in modules/network/* ; do
   module=$(basename $dir)
   echo Building network module $module...
-  docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/network/$module.log
+  if docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . ; then
+    echo Successfully built container for network $module
+  else
+    echo An error occured whilst building container for network module $module
+    exit 1
+  fi
 done
 
 # Build validators
@@ -56,7 +65,12 @@ mkdir -p build/devices
 for dir in modules/devices/* ; do
   module=$(basename $dir)
   echo Building validator module $module...
-  docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/devices/$module.log
+  if docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . ; then
+    echo Successfully built container for device module $module
+  else
+    echo An error occured whilst building container for device module $module
+    exit 1
+  fi
 done
 
 # Build test modules
@@ -65,7 +79,12 @@ mkdir -p build/test
 for dir in modules/test/* ; do
   module=$(basename $dir)
   echo Building test module $module...
-  docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . 2>&1 | tee build/test/$module.log
+  if docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . ; then
+    echo Successfully built container for test module $module
+  else
+    echo An error occured whilst building container for test module $module
+    exit 1
+  fi
 done
 
 echo Finished building modules

cmd/install

@@ -16,28 +16,52 @@
 
 echo Installing application dependencies
 
-TESTRUN_DIR=/usr/local/testrun
+# Collect command line arguments
+while getopts ":l" option; do
+   case $option in
+      l) # Install Testrun in local directory
+         TESTRUN_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd)
+   esac
+done
+
+# Check if TESTRUN_DIR has been set, otherwise install in /usr/local/testrun
+if [[ -z "${TESTRUN_DIR}" ]]; then
+  TESTRUN_DIR=/usr/local/testrun
+else
+  TESTRUN_DIR="${TESTRUN_DIR}"
+fi
+
+echo Installing Testrun at $TESTRUN_DIR
+
+# Create the folder if it doesn't exist
+mkdir -p $TESTRUN_DIR
 cd $TESTRUN_DIR
 
+# Activate the Python virtual environment
 python3 -m venv venv
-
 source venv/bin/activate
 
+# Install Python dependencies in virtual environment
 pip3 install -r framework/requirements.txt
 
 # Copy the default configuration
 cp -n local/system.json.example local/system.json
 
-# Set file permissions
+# Set file permissions on system config
 # This does not work on GitHub actions
 if logname ; then
   USER_NAME=$(logname)
   sudo chown "$USER_NAME" local/system.json
 fi
 
+# Exit out of python virtual environment
 deactivate
 
 # Build docker images
 sudo cmd/build
 
+# Create local folders
+mkdir -p local/devices
+mkdir -p local/root_certs
+
 echo Finished installing Testrun

cmd/prepare

@@ -19,6 +19,7 @@
 
 echo Installing system dependencies
 
+# Install system dependencies
 sudo apt-get update && sudo apt-get install openvswitch-common openvswitch-switch python3 libpangocairo-1.0-0
 
 echo Finished installing system dependencies

cmd/prune

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop any running containers
+echo Stopping any running containers
+running_containers=$(docker container ls -q --filter name=tr-*)
+if [ -n "$running_containers" ]; then
+  docker container kill $running_containers
+else
+  echo No containers were found running
+fi
+
+# Remove docker images
+echo Removing docker images
+docker_images=$(sudo docker images --filter=reference="test-run/*" -q)
+
+if [ -z "$docker_images" ]; then
+  echo No docker images to delete
+else
+  sudo docker rmi $docker_images > /dev/null
+fi
+
+# Remove docker networks
+echo Removing docker networks
+sudo docker network rm endev0 > /dev/null
+sudo docker network rm tr-private-net > /dev/null

framework/python/src/api/api.py

@@ -34,7 +34,7 @@
 DEVICE_MANUFACTURER_KEY = "manufacturer"
 DEVICE_MODEL_KEY = "model"
 DEVICE_TEST_MODULES_KEY = "test_modules"
-DEVICES_PATH = "/usr/local/testrun/local/devices"
+DEVICES_PATH = "local/devices"
 DEFAULT_DEVICE_INTF = "enx123456789123"
 
 LATEST_RELEASE_CHECK = ("https://api.github.com/repos/google/" +

framework/python/src/core/test_runner.py

@@ -67,13 +67,13 @@ def stop(self):
 
 def parse_args():
   parser = argparse.ArgumentParser(
-      description="Test Run",
+      description="Testrun",
       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
   parser.add_argument(
       "-f",
       "--config-file",
       default=None,
-      help="Define the configuration file for Test Run and Network Orchestrator"
+      help="Define the configuration file for Testrun and Network Orchestrator"
   )
   parser.add_argument(
       "--validate",

framework/python/src/test_orc/test_orchestrator.py

@@ -35,7 +35,7 @@
 SAVED_DEVICE_REPORTS = "report/{device_folder}/"
 LOCAL_DEVICE_REPORTS = "local/devices/{device_folder}/reports"
 DEVICE_ROOT_CERTS = "local/root_certs"
-TESTRUN_DIR = "/usr/local/testrun"
+API_URL = "http://localhost:8000"
 
 
 class TestOrchestrator:

testing/api/test_api

@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-TESTRUN_DIR=/usr/local/testrun
-
 ifconfig
 
 # Setup requirements
@@ -39,10 +37,10 @@ sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f  ./test
 sudo chown -R $USER local
 
 # Copy configuration to testrun
-sudo cp testing/api/system.json $TESTRUN_DIR/local/system.json
+sudo cp testing/api/system.json local/system.json
 
 # Needs to be sudo because this invokes bin/testrun 
-sudo $TESTRUN_DIR/venv/bin/python3 -m pytest -v testing/api/test_api.py
+sudo venv/bin/python3 -m pytest -v testing/api/test_api.py
 
 # Clean up network interfaces after use
 sudo docker network rm endev0