diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a89448fa2..2756be7ce 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,13 +17,10 @@ jobs: - name: Install dependencies shell: bash {0} run: cmd/prepare - - name: Package Testrun - shell: bash {0} - run: cmd/package - name: Install Testrun shell: bash {0} timeout-minutes: 10 - run: sudo dpkg -i testrun*.deb + run: TESTRUN_DIR=. cmd/install - name: Run baseline tests shell: bash {0} run: testing/baseline/test_baseline @@ -71,19 +68,16 @@ jobs: - name: Install dependencies shell: bash {0} run: cmd/prepare - - name: Package Testrun - shell: bash {0} - run: cmd/package - name: Install Testrun shell: bash {0} - run: sudo dpkg -i testrun*.deb + run: TESTRUN_DIR=. cmd/install timeout-minutes: 30 - name: Run tests shell: bash {0} run: testing/api/test_api - name: Archive runtime results if: ${{ always() }} - run: sudo tar --exclude-vcs -czf runtime.tgz /usr/local/testrun/runtime/ /usr/local/testrun/local/ + run: sudo tar --exclude-vcs -czf runtime.tgz runtime/ local/ - name: Upload runtime results uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 if: ${{ always() }} diff --git a/.gitignore b/.gitignore index bc014793a..0d985dd29 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ testing/unit/tls/output/ testing/unit/report/output/ *.deb -make/DEBIAN/postinst \ No newline at end of file +make/DEBIAN/postinst + +testrun.log \ No newline at end of file diff --git a/README.md b/README.md index ccd056c2e..0fbf71271 100644 --- a/README.md +++ b/README.md @@ -51,25 +51,30 @@ If the application has come across a problem at any point during setup or use, p The contributing requirements can be found in [CONTRIBUTING.md](CONTRIBUTING.md). In short, checkout the [Google CLA](https://cla.developers.google.com/) site to get started. ## FAQ :raising_hand: -1) What device networking functionality is validated by Testrun? +1) I have an issue whilst installing/upgrading Testrun, what do I do? + + Sometimes, issues may arise when installing or upgrading Testrun - this may happen due to one of many reasons due to the nature of the application. However, most of the time, it can be resolved by following a full Testrun re-install by using these commands: + - ```sudo docker system prune -a``` + - ```sudo apt install ./testrun-*.deb``` + +2) What device networking functionality is validated by Testrun? Best practices and requirements for IoT devices are constantly changing due to technological advances and discovery of vulnerabilities. The current expectations for IoT devices on Google deployments can be found in the [Application Security Requirements for IoT Devices](https://partner-security.withgoogle.com/docs/iot_requirements). Testrun aims to automate as much of the Application Security Requirements as possible. -2) What services are provided on the virtual network? +3) What services are provided on the virtual network? The following are network services that are containerized and accessible to the device under test though are likely to change over time: - DHCP in failover configuration with internet connectivity - IPv6 SLAAC - DNS - NTPv4 - - 802.1x Port Based Authentication -3) Can I run Testrun on a virtual machine? +4) Can I run Testrun on a virtual machine? Testrun can be virtualized if the 2x ethernet adapters are passed through to a Virtual Box VM as a USB device rather than managed network adapters. A full guide will be provided once virtualization of Testrun has been fully tested. - 4) Can I connect multiple devices to Testrun? + 5) Can I connect multiple devices to Testrun? In short, Yes you can. The way in which multiple devices could be tested simultaneously is yet to be decided. However, if you simply want to add field/peer devices during runtime (even another laptop performing manual testing) then you may connect the USB ethernet adapter to an unmanaged switch. diff --git a/bin/testrun b/bin/testrun index b58313bf9..079f8da7e 100755 --- a/bin/testrun +++ b/bin/testrun @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Check that user is root if [[ "$EUID" -ne 0 ]]; then echo "Must run as root. Use sudo $0" exit 1 @@ -22,11 +23,17 @@ fi # Ensure that /var/run/netns folder exists sudo mkdir -p /var/run/netns -export TESTRUNPATH=/usr/local/testrun -cd $TESTRUNPATH +# Get the directory of the running script +FILE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# Create device folder if it doesn't exist -mkdir -p local/devices +# Check if Testrun was installed as a package +if [ $FILE_PATH == "/usr/bin" ] ; then + export TESTRUN_PATH="/usr/local/testrun" +else + export TESTRUN_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd) +fi + +cd $TESTRUN_PATH # Remove existing runtime data rm -rf runtime/* @@ -35,7 +42,7 @@ rm -rf runtime/* source venv/bin/activate # Set the PYTHONPATH to include the "src" directory -export PYTHONPATH="$TESTRUNPATH/framework/python/src" +export PYTHONPATH="$TESTRUN_PATH/framework/python/src" python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log deactivate \ No newline at end of file diff --git a/cmd/build b/cmd/build index da67db00a..d15171f31 100755 --- a/cmd/build +++ b/cmd/build @@ -38,8 +38,12 @@ echo Building docker images # Build user interface echo Building user interface -mkdir -p build/ui -docker build -t test-run/ui -f modules/ui/ui.Dockerfile . 2>&1 | tee build/ui/ui.log +if docker build -t test-run/ui -f modules/ui/ui.Dockerfile . ; then + echo Successully built the user interface +else + echo An error occured whilst building the user interface + exit 1 +fi # Build network modules echo Building network modules @@ -47,7 +51,12 @@ mkdir -p build/network for dir in modules/network/* ; do module=$(basename $dir) echo Building network module $module... - docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/network/$module.log + if docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . ; then + echo Successfully built container for network $module + else + echo An error occured whilst building container for network module $module + exit 1 + fi done # Build validators @@ -56,7 +65,12 @@ mkdir -p build/devices for dir in modules/devices/* ; do module=$(basename $dir) echo Building validator module $module... - docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/devices/$module.log + if docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . ; then + echo Successfully built container for device module $module + else + echo An error occured whilst building container for device module $module + exit 1 + fi done # Build test modules @@ -65,7 +79,12 @@ mkdir -p build/test for dir in modules/test/* ; do module=$(basename $dir) echo Building test module $module... - docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . 2>&1 | tee build/test/$module.log + if docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . ; then + echo Successfully built container for test module $module + else + echo An error occured whilst building container for test module $module + exit 1 + fi done echo Finished building modules \ No newline at end of file diff --git a/cmd/install b/cmd/install index c4f53b905..9d4f4de52 100755 --- a/cmd/install +++ b/cmd/install @@ -16,28 +16,52 @@ echo Installing application dependencies -TESTRUN_DIR=/usr/local/testrun +# Collect command line arguments +while getopts ":l" option; do + case $option in + l) # Install Testrun in local directory + TESTRUN_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd) + esac +done + +# Check if TESTRUN_DIR has been set, otherwise install in /usr/local/testrun +if [[ -z "${TESTRUN_DIR}" ]]; then + TESTRUN_DIR=/usr/local/testrun +else + TESTRUN_DIR="${TESTRUN_DIR}" +fi + +echo Installing Testrun at $TESTRUN_DIR + +# Create the folder if it doesn't exist +mkdir -p $TESTRUN_DIR cd $TESTRUN_DIR +# Activate the Python virtual environment python3 -m venv venv - source venv/bin/activate +# Install Python dependencies in virtual environment pip3 install -r framework/requirements.txt # Copy the default configuration cp -n local/system.json.example local/system.json -# Set file permissions +# Set file permissions on system config # This does not work on GitHub actions if logname ; then USER_NAME=$(logname) sudo chown "$USER_NAME" local/system.json fi +# Exit out of python virtual environment deactivate # Build docker images sudo cmd/build +# Create local folders +mkdir -p local/devices +mkdir -p local/root_certs + echo Finished installing Testrun diff --git a/cmd/prepare b/cmd/prepare index 436af87ae..252d505a0 100755 --- a/cmd/prepare +++ b/cmd/prepare @@ -19,6 +19,7 @@ echo Installing system dependencies +# Install system dependencies sudo apt-get update && sudo apt-get install openvswitch-common openvswitch-switch python3 libpangocairo-1.0-0 echo Finished installing system dependencies diff --git a/cmd/prune b/cmd/prune new file mode 100755 index 000000000..ed9b4851d --- /dev/null +++ b/cmd/prune @@ -0,0 +1,39 @@ +#!/bin/bash -e + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Stop any running containers +echo Stopping any running containers +running_containers=$(docker container ls -q --filter name=tr-*) +if [ -n "$running_containers" ]; then + docker container kill $running_containers +else + echo No containers were found running +fi + +# Remove docker images +echo Removing docker images +docker_images=$(sudo docker images --filter=reference="test-run/*" -q) + +if [ -z "$docker_images" ]; then + echo No docker images to delete +else + sudo docker rmi $docker_images > /dev/null +fi + +# Remove docker networks +echo Removing docker networks +sudo docker network rm endev0 > /dev/null +sudo docker network rm tr-private-net > /dev/null \ No newline at end of file diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 43efeb593..e6cdefb8b 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -34,7 +34,7 @@ DEVICE_MANUFACTURER_KEY = "manufacturer" DEVICE_MODEL_KEY = "model" DEVICE_TEST_MODULES_KEY = "test_modules" -DEVICES_PATH = "/usr/local/testrun/local/devices" +DEVICES_PATH = "local/devices" DEFAULT_DEVICE_INTF = "enx123456789123" LATEST_RELEASE_CHECK = ("https://api.github.com/repos/google/" + diff --git a/framework/python/src/core/test_runner.py b/framework/python/src/core/test_runner.py index aa0668a66..07b6121b5 100644 --- a/framework/python/src/core/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -67,13 +67,13 @@ def stop(self): def parse_args(): parser = argparse.ArgumentParser( - description="Test Run", + description="Testrun", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( "-f", "--config-file", default=None, - help="Define the configuration file for Test Run and Network Orchestrator" + help="Define the configuration file for Testrun and Network Orchestrator" ) parser.add_argument( "--validate", diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 062a3ca9c..01ffb5845 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -35,7 +35,7 @@ SAVED_DEVICE_REPORTS = "report/{device_folder}/" LOCAL_DEVICE_REPORTS = "local/devices/{device_folder}/reports" DEVICE_ROOT_CERTS = "local/root_certs" -TESTRUN_DIR = "/usr/local/testrun" +API_URL = "http://localhost:8000" class TestOrchestrator: diff --git a/testing/api/test_api b/testing/api/test_api index 4b379c717..d86ff42a9 100755 --- a/testing/api/test_api +++ b/testing/api/test_api @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -TESTRUN_DIR=/usr/local/testrun - ifconfig # Setup requirements @@ -39,10 +37,10 @@ sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f ./test sudo chown -R $USER local # Copy configuration to testrun -sudo cp testing/api/system.json $TESTRUN_DIR/local/system.json +sudo cp testing/api/system.json local/system.json # Needs to be sudo because this invokes bin/testrun -sudo $TESTRUN_DIR/venv/bin/python3 -m pytest -v testing/api/test_api.py +sudo venv/bin/python3 -m pytest -v testing/api/test_api.py # Clean up network interfaces after use sudo docker network rm endev0 diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 44daef335..f0e9ac51c 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -37,7 +37,6 @@ LOG_PATH = "/tmp/testrun.log" TEST_SITE_DIR = ".." -TESTRUN_DIR = "/usr/local/testrun" DEVICES_DIRECTORY = "local/devices" TESTING_DEVICES = "../device_configs" SYSTEM_CONFIG_PATH = "local/system.json" @@ -112,7 +111,7 @@ def testing_devices(): local_delete_devices(ALL_DEVICES) shutil.copytree( os.path.join(os.path.dirname(__file__), TESTING_DEVICES), - os.path.join(TESTRUN_DIR, DEVICES_DIRECTORY), + os.path.join(DEVICES_DIRECTORY), dirs_exist_ok=True, ) return local_get_devices() @@ -123,7 +122,7 @@ def testrun(request): """ Start intstance of testrun """ test_name = request.node.originalname proc = subprocess.Popen( - "testrun", + "bin/testrun", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", @@ -209,8 +208,7 @@ def get_network_interfaces(): def local_delete_devices(path): """ Deletes all local devices """ - devices_path = os.path.join(TESTRUN_DIR, DEVICES_DIRECTORY) - for thing in Path(devices_path).glob(path): + for thing in Path(DEVICES_DIRECTORY).glob(path): if thing.is_file(): thing.unlink() else: @@ -220,7 +218,7 @@ def local_delete_devices(path): def local_get_devices(): """ Returns path to device configs of devices in local/devices directory""" return sorted( - Path(os.path.join(TESTRUN_DIR, DEVICES_DIRECTORY)).glob( + Path(DEVICES_DIRECTORY).glob( "*/device_config.json" ) ) @@ -240,7 +238,7 @@ def test_get_system_interfaces(testrun): def test_modify_device(testing_devices, testrun): with open( os.path.join( - TESTRUN_DIR, DEVICES_DIRECTORY, testing_devices[1] + DEVICES_DIRECTORY, testing_devices[1] ) ) as f: local_device = json.load(f) @@ -350,7 +348,7 @@ def test_create_get_devices(empty_devices_dir, testrun): def test_get_system_config(testrun): r = requests.get(f"{API}/system/config") - with open(os.path.join(TESTRUN_DIR, SYSTEM_CONFIG_PATH)) as f: + with open(SYSTEM_CONFIG_PATH) as f: local_config = json.load(f) api_config = json.loads(r.text) diff --git a/testing/baseline/test_baseline b/testing/baseline/test_baseline index c52c08aef..f64c580ba 100755 --- a/testing/baseline/test_baseline +++ b/testing/baseline/test_baseline @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -TESTRUN_OUT=/tmp/testrun.log -TESTRUN_DIR=/usr/local/testrun +TESTRUN_OUT=testrun.log ifconfig @@ -38,12 +37,12 @@ sudo /usr/share/openvswitch/scripts/ovs-ctl start sudo docker build ./testing/docker/ci_baseline -t ci1 -f ./testing/docker/ci_baseline/Dockerfile # Copy configuration to testrun -sudo cp testing/baseline/system.json $TESTRUN_DIR/local/system.json +sudo cp testing/baseline/system.json local/system.json # Copy device configs to testrun -sudo cp -r testing/device_configs/* $TESTRUN_DIR/local/devices +sudo cp -r testing/device_configs/* local/devices -sudo testrun --single-intf --no-ui --validate > $TESTRUN_OUT 2>&1 & +sudo bin/testrun --single-intf --no-ui --validate > $TESTRUN_OUT 2>&1 & TPID=$! # Time to wait for testrun to be ready