From 2a9d32cec3872341c6827b20478640cd3240a603 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 23 Oct 2024 16:05:44 -0600 Subject: [PATCH 1/7] Add target and firmware args for no-ui option --- framework/python/src/core/test_runner.py | 28 +++++++++++++++++++++--- framework/python/src/core/testrun.py | 14 +++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/framework/python/src/core/test_runner.py b/framework/python/src/core/test_runner.py index a295f47e1..830b665a9 100644 --- a/framework/python/src/core/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -37,13 +37,17 @@ def __init__(self, validate=False, net_only=False, single_intf=False, - no_ui=False): + no_ui=False, + target=None, + firmware=None): self._register_exits() self._testrun = Testrun(config_file=config_file, validate=validate, net_only=net_only, single_intf=single_intf, - no_ui=no_ui) + no_ui=no_ui, + target_mac=target, + firmware=firmware) def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -91,7 +95,23 @@ def parse_args(): default=False, action="store_true", help="Do not launch the user interface") + parser.add_argument("--target", + default=None, + type=str, + help="MAC address of the target device") + parser.add_argument("-fw", + "--firmware", + default=None, + type=str, + help="Firmware version to be tested") + + parsed_args = parser.parse_known_args()[0] + + if (parsed_args.no_ui + and (parsed_args.target is None or parsed_args.firmware is None)): + parser.error("--target and --firmware required when --no-ui is specified") + return parsed_args @@ -101,4 +121,6 @@ def parse_args(): validate=args.validate, net_only=args.net_only, single_intf=args.single_intf, - no_ui=args.no_ui) + no_ui=args.no_ui, + target=args.target, + firmware=args.firmware) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index ace0c792a..f31bd4f98 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -67,7 +67,9 @@ def __init__(self, validate=False, net_only=False, single_intf=False, - no_ui=False): + no_ui=False, + target_mac=None, + firmware=None): # Locate parent directory current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -112,6 +114,16 @@ def __init__(self, # Load device repository self.load_all_devices() + # If no_ui selected and not network only mode, + # load the target device into the session + if self._no_ui and not net_only: + target_device = self._session.get_device(target_mac) + if target_device is not None: + target_device.firmware=firmware + self._session.set_target_device(target_device) + else: + raise Exception('Target device specified does not exist in device registry') + # Load test modules self._test_orc.start() From e0c3a9b96f56ac3c752d9da739f22f146eaae1a0 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 23 Oct 2024 16:09:32 -0600 Subject: [PATCH 2/7] pylint fixes --- framework/python/src/core/test_runner.py | 9 +- framework/python/src/core/testrun.py | 115 +++++++++-------------- 2 files changed, 48 insertions(+), 76 deletions(-) diff --git a/framework/python/src/core/test_runner.py b/framework/python/src/core/test_runner.py index 830b665a9..64b77755d 100644 --- a/framework/python/src/core/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Wrapper for the Testrun that simplifies virtual testing procedure by allowing direct calling from the command line. @@ -77,8 +76,7 @@ def parse_args(): "-f", "--config-file", default=None, - help="Define the configuration file for Testrun and Network Orchestrator" - ) + help="Define the configuration file for Testrun and Network Orchestrator") parser.add_argument( "--validate", default=False, @@ -105,11 +103,10 @@ def parse_args(): type=str, help="Firmware version to be tested") - parsed_args = parser.parse_known_args()[0] - if (parsed_args.no_ui - and (parsed_args.target is None or parsed_args.firmware is None)): + if (parsed_args.no_ui + and (parsed_args.target is None or parsed_args.firmware is None)): parser.error("--target and --firmware required when --no-ui is specified") return parsed_args diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index f31bd4f98..02a2c5641 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """The overall control of the Testrun application. This file provides the integration between all of the Testrun components, such as net_orc, test_orc and test_ui. @@ -55,6 +54,7 @@ MAX_DEVICE_REPORTS_KEY = 'max_device_reports' + class Testrun: # pylint: disable=too-few-public-methods """Testrun controller. @@ -76,8 +76,8 @@ def __init__(self, # Locate the test-run root directory, 4 levels, # src->python->framework->test-run - self._root_dir = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(current_dir)))) + self._root_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) # Determine config file if config_file is None: @@ -105,24 +105,22 @@ def __init__(self, if validate: self._session.add_runtime_param('validate') - self._net_orc = net_orc.NetworkOrchestrator( - session=self._session) - self._test_orc = test_orc.TestOrchestrator( - self._session, - self._net_orc) + self._net_orc = net_orc.NetworkOrchestrator(session=self._session) + self._test_orc = test_orc.TestOrchestrator(self._session, self._net_orc) # Load device repository self.load_all_devices() - # If no_ui selected and not network only mode, + # If no_ui selected and not network only mode, # load the target device into the session if self._no_ui and not net_only: target_device = self._session.get_device(target_mac) if target_device is not None: - target_device.firmware=firmware + target_device.firmware = firmware self._session.set_target_device(target_device) else: - raise Exception('Target device specified does not exist in device registry') + raise ValueError( + 'Target device specified does not exist in device registry') # Load test modules self._test_orc.start() @@ -177,8 +175,7 @@ def _load_devices(self, device_dir): for device_folder in os.listdir(device_dir): - device_config_file_path = os.path.join(device_dir, - device_folder, + device_config_file_path = os.path.join(device_dir, device_folder, DEVICE_CONFIG) # Check if device config file exists before loading @@ -195,7 +192,7 @@ def _load_devices(self, device_dir): device_config_json = json.load(device_config_file) except json.decoder.JSONDecodeError as e: LOGGER.error('Invalid JSON found in ' + - f'device configuration {device_config_file_path}') + f'device configuration {device_config_file_path}') LOGGER.debug(e) continue @@ -231,11 +228,11 @@ def _load_devices(self, device_dir): if DEVICE_ADDITIONAL_INFO_KEY in device_config_json: device.additional_info = device_config_json.get( - DEVICE_ADDITIONAL_INFO_KEY) + DEVICE_ADDITIONAL_INFO_KEY) if None in [device.type, device.technology, device.test_pack]: LOGGER.warning( - 'Device is outdated and requires further configuration') + 'Device is outdated and requires further configuration') device.status = 'Invalid' self._load_test_reports(device) @@ -262,26 +259,20 @@ def _load_test_reports(self, device): for report_folder in os.listdir(reports_folder): # 1.3 file path - report_json_file_path = os.path.join( - reports_folder, - report_folder, - 'test', - device.mac_addr.replace(':',''), - 'report.json') + report_json_file_path = os.path.join(reports_folder, report_folder, + 'test', + device.mac_addr.replace(':', ''), + 'report.json') if not os.path.isfile(report_json_file_path): # Revert to pre 1.3 file path - report_json_file_path = os.path.join( - reports_folder, - report_folder, - 'report.json') + report_json_file_path = os.path.join(reports_folder, report_folder, + 'report.json') if not os.path.isfile(report_json_file_path): # Revert to pre 1.3 file path - report_json_file_path = os.path.join( - reports_folder, - report_folder, - 'report.json') + report_json_file_path = os.path.join(reports_folder, report_folder, + 'report.json') # Check if the report.json file exists if not os.path.isfile(report_json_file_path): @@ -297,9 +288,8 @@ def _load_test_reports(self, device): def get_reports_folder(self, device): """Return the reports folder path for the device""" - return os.path.join(self._root_dir, - LOCAL_DEVICES_DIR, - device.device_folder, 'reports') + return os.path.join(self._root_dir, LOCAL_DEVICES_DIR, device.device_folder, + 'reports') def delete_report(self, device: Device, timestamp): LOGGER.debug(f'Deleting test report for device {device.model} ' + @@ -320,15 +310,13 @@ def delete_report(self, device: Device, timestamp): def create_device(self, device: Device): # Define the device folder location - device_folder_path = os.path.join(self._root_dir, - LOCAL_DEVICES_DIR, + device_folder_path = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, device.device_folder) # Create the directory os.makedirs(device_folder_path) - config_file_path = os.path.join(device_folder_path, - DEVICE_CONFIG) + config_file_path = os.path.join(device_folder_path, DEVICE_CONFIG) with open(config_file_path, 'w', encoding='utf-8') as config_file: config_file.writelines(json.dumps(device.to_config_json(), indent=4)) @@ -345,10 +333,8 @@ def save_device(self, device: Device): """Edit and save an existing device config.""" # Obtain the config file path - config_file_path = os.path.join(self._root_dir, - LOCAL_DEVICES_DIR, - device.device_folder, - DEVICE_CONFIG) + config_file_path = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, + device.device_folder, DEVICE_CONFIG) with open(config_file_path, 'w+', encoding='utf-8') as config_file: config_file.writelines(json.dumps(device.to_config_json(), indent=4)) @@ -361,9 +347,8 @@ def save_device(self, device: Device): def delete_device(self, device: Device): # Obtain the config file path - device_folder = os.path.join(self._root_dir, - LOCAL_DEVICES_DIR, - device.device_folder) + device_folder = os.path.join(self._root_dir, LOCAL_DEVICES_DIR, + device.device_folder) # Delete the device directory shutil.rmtree(device_folder) @@ -378,17 +363,13 @@ def start(self): self._start_network() self.get_net_orc().get_listener().register_callback( - self._device_discovered, - [NetworkEvent.DEVICE_DISCOVERED] - ) + self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED]) if self._net_only: LOGGER.info('Network only option configured, no tests will be run') else: self.get_net_orc().get_listener().register_callback( - self._device_stable, - [NetworkEvent.DEVICE_STABLE] - ) + self._device_stable, [NetworkEvent.DEVICE_STABLE]) self.get_net_orc().start_listener() LOGGER.info('Waiting for devices on the network...') @@ -523,16 +504,12 @@ def start_ui(self): client = docker.from_env() try: - client.containers.run( - image='testrun/ui', - auto_remove=True, - name='tr-ui', - hostname='testrun.io', - detach=True, - ports={ - '80': 8080 - } - ) + client.containers.run(image='testrun/ui', + auto_remove=True, + name='tr-ui', + hostname='testrun.io', + detach=True, + ports={'80': 8080}) except ImageNotFound as ie: LOGGER.error('An error occured whilst starting the UI. ' + 'Please investigate and try again.') @@ -561,16 +538,14 @@ def start_ws(self): client = docker.from_env() try: - client.containers.run( - image='testrun/ws', - auto_remove=True, - name='tr-ws', - detach=True, - ports={ - '9001': 9001, - '1883': 1883 - } - ) + client.containers.run(image='testrun/ws', + auto_remove=True, + name='tr-ws', + detach=True, + ports={ + '9001': 9001, + '1883': 1883 + }) except ImageNotFound as ie: LOGGER.error('An error occured whilst starting the websockets server. ' + 'Please investigate and try again.') From ee89b557da83e387a7057a7bfbd673cdd99f34b4 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 23 Oct 2024 16:30:37 -0600 Subject: [PATCH 3/7] Add network only filter --- framework/python/src/core/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/core/test_runner.py b/framework/python/src/core/test_runner.py index 64b77755d..4fd555290 100644 --- a/framework/python/src/core/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -105,7 +105,7 @@ def parse_args(): parsed_args = parser.parse_known_args()[0] - if (parsed_args.no_ui + if (parsed_args.no_ui and not parsed_args.net_only and (parsed_args.target is None or parsed_args.firmware is None)): parser.error("--target and --firmware required when --no-ui is specified") From 1c4cf622892369d51fe9eb7d3216f10ff072a61b Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 23 Oct 2024 17:03:34 -0600 Subject: [PATCH 4/7] Add new vars to baseline test --- testing/baseline/test_baseline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/baseline/test_baseline b/testing/baseline/test_baseline index 4ab0d75e8..44a17d348 100755 --- a/testing/baseline/test_baseline +++ b/testing/baseline/test_baseline @@ -40,7 +40,7 @@ sudo cp testing/baseline/system.json local/system.json # Copy device configs to testrun sudo cp -r testing/device_configs/* local/devices -sudo bin/testrun --single-intf --no-ui --validate > $TESTRUN_OUT 2>&1 & +sudo bin/testrun --single-intf --no-ui --target 02:42:aa:00:01:01 -fw 1.0 --validate > $TESTRUN_OUT 2>&1 & TPID=$! # Time to wait for testrun to be ready From fc8b1b2839a57e9eda126d865eafaa56c8bc8f70 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 25 Oct 2024 15:50:40 -0600 Subject: [PATCH 5/7] Cleanup arg error message --- bin/testrun | 94 ++++++++++++------------ framework/python/src/core/test_runner.py | 19 ++++- 2 files changed, 65 insertions(+), 48 deletions(-) diff --git a/bin/testrun b/bin/testrun index 079f8da7e..b5c8d80eb 100755 --- a/bin/testrun +++ b/bin/testrun @@ -1,48 +1,48 @@ -#!/bin/bash -e - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Check that user is root -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo $0" - exit 1 -fi - -# Ensure that /var/run/netns folder exists -sudo mkdir -p /var/run/netns - -# Get the directory of the running script -FILE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# Check if Testrun was installed as a package -if [ $FILE_PATH == "/usr/bin" ] ; then - export TESTRUN_PATH="/usr/local/testrun" -else - export TESTRUN_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd) -fi - -cd $TESTRUN_PATH - -# Remove existing runtime data -rm -rf runtime/* - -# Activate Python virtual environment -source venv/bin/activate - -# Set the PYTHONPATH to include the "src" directory -export PYTHONPATH="$TESTRUN_PATH/framework/python/src" -python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log - +#!/bin/bash -e + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Check that user is root +if [[ "$EUID" -ne 0 ]]; then + echo "Must run as root. Use sudo $0" + exit 1 +fi + +# Ensure that /var/run/netns folder exists +sudo mkdir -p /var/run/netns + +# Get the directory of the running script +FILE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Check if Testrun was installed as a package +if [ $FILE_PATH == "/usr/bin" ] ; then + export TESTRUN_PATH="/usr/local/testrun" +else + export TESTRUN_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd) +fi + +cd $TESTRUN_PATH + +# Remove existing runtime data +rm -rf runtime/* + +# Activate Python virtual environment +source venv/bin/activate + +# Set the PYTHONPATH to include the "src" directory +export PYTHONPATH="$TESTRUN_PATH/framework/python/src" +python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log + deactivate \ No newline at end of file diff --git a/framework/python/src/core/test_runner.py b/framework/python/src/core/test_runner.py index 4fd555290..0a8f69e43 100644 --- a/framework/python/src/core/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -24,6 +24,7 @@ from testrun import Testrun from common import logger import signal +import io LOGGER = logger.get_logger("runner") @@ -107,7 +108,23 @@ def parse_args(): if (parsed_args.no_ui and not parsed_args.net_only and (parsed_args.target is None or parsed_args.firmware is None)): - parser.error("--target and --firmware required when --no-ui is specified") + # Capture help text + help_text = io.StringIO() + parser.print_help(file=help_text) + + # Get help text as lines and find where "Testrun" starts (skip usage) + help_lines = help_text.getvalue().splitlines() + start_index = next( + (i for i, line in enumerate(help_lines) if "Testrun" in line), 0) + + # Join only lines starting from "Testrun" and print without extra newlines + help_message = "\n".join(line.rstrip() for line in help_lines[start_index:]) + print(help_message) + + print( + "Error: --target and --firmware are required when --no-ui is specified", + file=sys.stderr) + sys.exit(1) return parsed_args From 241648b8bc55eef87cf1fc2a465a07223139e50c Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 25 Oct 2024 15:56:45 -0600 Subject: [PATCH 6/7] Update bad target error messaging --- framework/python/src/core/testrun.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 02a2c5641..5d4e78e9c 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -119,8 +119,11 @@ def __init__(self, target_device.firmware = firmware self._session.set_target_device(target_device) else: - raise ValueError( - 'Target device specified does not exist in device registry') + print( + f'Target device specified does not exist in device registry: ' + f'{target_mac}', + file=sys.stderr) + sys.exit(1) # Load test modules self._test_orc.start() From 86c05c5715aa18bd4d5496573fe3b5982c849b50 Mon Sep 17 00:00:00 2001 From: Marius <86727846+MariusBaldovin@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:29:25 +0000 Subject: [PATCH 7/7] converted bin/testrun back to unix format (#928) --- bin/testrun | 94 ++++++++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/bin/testrun b/bin/testrun index b5c8d80eb..079f8da7e 100755 --- a/bin/testrun +++ b/bin/testrun @@ -1,48 +1,48 @@ -#!/bin/bash -e - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Check that user is root -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo $0" - exit 1 -fi - -# Ensure that /var/run/netns folder exists -sudo mkdir -p /var/run/netns - -# Get the directory of the running script -FILE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# Check if Testrun was installed as a package -if [ $FILE_PATH == "/usr/bin" ] ; then - export TESTRUN_PATH="/usr/local/testrun" -else - export TESTRUN_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd) -fi - -cd $TESTRUN_PATH - -# Remove existing runtime data -rm -rf runtime/* - -# Activate Python virtual environment -source venv/bin/activate - -# Set the PYTHONPATH to include the "src" directory -export PYTHONPATH="$TESTRUN_PATH/framework/python/src" -python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log - +#!/bin/bash -e + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Check that user is root +if [[ "$EUID" -ne 0 ]]; then + echo "Must run as root. Use sudo $0" + exit 1 +fi + +# Ensure that /var/run/netns folder exists +sudo mkdir -p /var/run/netns + +# Get the directory of the running script +FILE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Check if Testrun was installed as a package +if [ $FILE_PATH == "/usr/bin" ] ; then + export TESTRUN_PATH="/usr/local/testrun" +else + export TESTRUN_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")"/.. && pwd) +fi + +cd $TESTRUN_PATH + +# Remove existing runtime data +rm -rf runtime/* + +# Activate Python virtual environment +source venv/bin/activate + +# Set the PYTHONPATH to include the "src" directory +export PYTHONPATH="$TESTRUN_PATH/framework/python/src" +python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log + deactivate \ No newline at end of file