From e4b55a09341cce96712bea3938d5df73018a2eeb Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Fri, 4 Aug 2023 14:42:21 +0100 Subject: [PATCH 1/4] Add /history and device config endpoints --- framework/python/src/api/api.py | 20 ++-- framework/python/src/common/device.py | 26 ++++- framework/python/src/common/session.py | 31 ++++-- framework/python/src/common/testreport.py | 84 ++++++++++++++++ framework/python/src/core/testrun.py | 99 +++++++++++++++++-- .../python/src/test_orc/test_orchestrator.py | 6 +- 6 files changed, 239 insertions(+), 27 deletions(-) create mode 100644 framework/python/src/common/testreport.py diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index f63f1825a..c2e22d4c6 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -48,7 +48,7 @@ def __init__(self, test_run): self._router.add_api_route("/system/stop", self.stop_test_run, methods=["POST"]) self._router.add_api_route("/system/status", self.get_status) - + self._router.add_api_route("/history", self.get_history) self._router.add_api_route("/devices", self.get_devices) self._router.add_api_route("/device", self.save_device, methods=["POST"]) @@ -160,7 +160,8 @@ async def get_status(self): return self._test_run.get_session().to_json() async def get_history(self): - LOGGER.info("Returning previous Test Runs to UI") + LOGGER.debug("Received history list request") + return self._session.get_all_reports() async def save_device(self, request: Request, response: Response): LOGGER.debug("Received device post request") @@ -174,18 +175,25 @@ async def save_device(self, request: Request, response: Response): return self._generate_msg(False, "Invalid request received") device = self._session.get_device(device_json.get(DEVICE_MAC_ADDR_KEY)) + if device is None: + # Create new device device = Device() device.mac_addr = device_json.get(DEVICE_MAC_ADDR_KEY) + device.manufacturer = device_json.get(DEVICE_MANUFACTURER_KEY) + device.model = device_json.get(DEVICE_MODEL_KEY) + device.device_folder = device.manufacturer + " " + device.model + + self._test_run.create_device(device) response.status_code = status.HTTP_201_CREATED - device.manufacturer = device_json.get(DEVICE_MANUFACTURER_KEY) - device.model = device_json.get(DEVICE_MODEL_KEY) + else: - self._session.save_device(device) + self._test_run.save_device(device, device_json) + response.status_code = status.HTTP_200_OK - return device + return device.to_config_json() # Catch JSON Decode error etc except JSONDecodeError: diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index e2552d75a..2d9377a8b 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -14,7 +14,8 @@ """Track device object information.""" -from dataclasses import dataclass +from typing import Dict +from dataclasses import dataclass, field @dataclass class Device(): @@ -23,13 +24,24 @@ class Device(): mac_addr: str = None manufacturer: str = None model: str = None - test_modules: str = None + test_modules: Dict = field(default_factory=dict) ip_addr: str = None firmware: str = None device_folder: str = None + reports = [] max_device_reports: int = None + def add_report(self, report): + self.reports.append(report) + + def get_reports(self): + return self.reports + + # TODO: Add ability to remove reports once test reports have been cleaned up + def to_json(self): + """Returns the device as a python dictionary. This is used for the + # system status API endpoint and in the report.""" device_json = {} device_json['mac_addr'] = self.mac_addr device_json['manufacturer'] = self.manufacturer @@ -37,3 +49,13 @@ def to_json(self): if self.firmware is not None: device_json['firmware'] = self.firmware return device_json + + def to_config_json(self): + """Returns the device as a python disctionary. Fields relevant to the device + config json file are exported.""" + device_json = {} + device_json['mac_addr'] = self.mac_addr + device_json['manufacturer'] = self.manufacturer + device_json['model'] = self.model + device_json['test_modules'] = self.test_modules + return device_json diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 13e4b09fb..09e8ca5ed 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -37,16 +37,15 @@ def __init__(self, config_file): self._finished = None self._results = [] self._runtime_params = [] + self._device_repository = [] self._config_file = config_file - self._config = self._get_default_config() self._load_config() - self._device_repository = [] - def start(self): - self._status = 'Starting' + self.reset() + self._status = 'Waiting for device' self._started = datetime.datetime.now() def get_started(self): @@ -131,7 +130,7 @@ def get_monitor_period(self): def get_startup_timeout(self): return self._config.get(STARTUP_TIMEOUT_KEY) - + def get_max_device_reports(self): return self._config.get(MAX_DEVICE_REPORTS_KEY) @@ -157,10 +156,6 @@ def get_device(self, mac_addr): return device return None - def save_device(self, device): - # TODO: We need to save the folder path of the device config - return - def get_status(self): return self._status @@ -173,6 +168,17 @@ def get_test_results(self): def add_test_result(self, test_result): self._results.append(test_result) + def get_all_reports(self): + + reports = [] + + for device in self.get_device_repository(): + device_reports = device.get_reports() + for device_report in device_reports: + reports.append(device_report.to_json()) + + return reports + def reset(self): self.set_status('Idle') self.set_target_device(None) @@ -181,10 +187,15 @@ def reset(self): self._finished = None def to_json(self): - return { + + # TODO: Add report URL + + session_json = { 'status': self.get_status(), 'device': self.get_target_device(), 'started': self.get_started(), 'finished': self.get_finished(), 'results': self.get_test_results() } + + return session_json diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py new file mode 100644 index 000000000..ba35ff27a --- /dev/null +++ b/framework/python/src/common/testreport.py @@ -0,0 +1,84 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Store previous test run information.""" + +from datetime import datetime + +DATE_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' + +class TestReport(): + """Represents a previous Test Run report.""" + + def __init__(self, + status='Non-Compliant', + started=None, + finished=None, + total_tests=0 + ): + self._device = {} + self._status: str = status + self._started = started + self._finished = finished + self._total_tests = total_tests + self._results = [] + + def get_status(self): + return self._status + + def get_started(self): + return self._started + + def get_finished(self): + return self._finished + + def get_duration_seconds(self): + diff = self._finished - self._started + return diff.total_seconds() + + def get_duration(self): + return str(datetime.timedelta(seconds=self.get_duration_seconds())) + + def add_test(self, test): + self._results.append(test) + + def to_json(self): + report_json = {} + report_json['device'] = self._device + report_json['status'] = self._status + report_json['started'] = self._started.strftime(DATE_TIME_FORMAT) + report_json['finished'] = self._finished.strftime(DATE_TIME_FORMAT) + report_json['tests'] = {'total': self._total_tests, + 'results': self._results} + return report_json + + def from_json(self, json_file): + + self._device['mac_addr'] = json_file['device']['mac_addr'] + self._device['manufacturer'] = json_file['device']['manufacturer'] + self._device['model'] = json_file['device']['model'] + + if 'firmware' in self._device: + self._device['firmware'] = json_file['device']['firmware'] + + self._status = json_file['status'] + self._started = datetime.strptime(json_file['started'], DATE_TIME_FORMAT) + self._finished = datetime.strptime(json_file['finished'], DATE_TIME_FORMAT) + self._total_tests = json_file['tests']['total'] + + # Loop through test results + for test_result in json_file['tests']['results']: + self.add_test(test_result) + + return self diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 6e3a6da5d..7b89f46b0 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -28,6 +28,7 @@ from common import logger, util from common.device import Device from common.session import TestRunSession +from common.testreport import TestReport from api.api import Api from net_orc.listener import NetworkEvent from net_orc import network_orchestrator as net_orc @@ -108,8 +109,11 @@ def __init__(self, self.start() else: + + # Build UI image self._api = Api(self) self._api.start() + # Start UI container # Hold until API ends while True: @@ -117,7 +121,10 @@ def __init__(self, def _load_all_devices(self): self._load_devices(device_dir=LOCAL_DEVICES_DIR) - self._load_devices(device_dir=RESOURCE_DEVICES_DIR) + + # Temporarily removing loading of template device + # configs (feature not required yet) + # self._load_devices(device_dir=RESOURCE_DEVICES_DIR) return self.get_session().get_device_repository() def _load_devices(self, device_dir): @@ -130,10 +137,13 @@ def _load_devices(self, device_dir): device_config_file_path = os.path.join(device_dir, device_folder, DEVICE_CONFIG) + + # Check if device config file exists before loading if not os.path.exists(device_config_file_path): LOGGER.error(f'Device configuration file missing from device {device_folder}') continue + # Open device config file with open(device_config_file_path, encoding='utf-8') as device_config_file: device_config_json = json.load(device_config_file) @@ -152,10 +162,90 @@ def _load_devices(self, device_dir): test_modules=test_modules, max_device_reports=max_device_reports, device_folder=device_folder) - self.get_session().add_device(device) + # Load reports for this device + self._load_test_reports(device) + + # Add device to device repository self.get_session().add_device(device) - LOGGER.debug(f'Loaded device {device.manufacturer} {device.model} with MAC address {device.mac_addr}') + LOGGER.debug(f'Loaded device {device.manufacturer} ' + + f'{device.model} with MAC address {device.mac_addr}') + + def _load_test_reports(self, device: Device): + + LOGGER.debug(f'Loading test reports for device {device.model}') + + # Locate reports folder + reports_folder = os.path.join(root_dir, + LOCAL_DEVICES_DIR, + device.device_folder, 'reports') + + # Check if reports folder exists (device may have no reports) + if not os.path.exists(reports_folder): + return + + for report_folder in os.listdir(reports_folder): + report_json_file_path = os.path.join( + reports_folder, + report_folder, + 'report.json') + + # Check if the report.json file exists + if not os.path.isfile(report_json_file_path): + # Some error may have occured during this test run + continue + + with open(report_json_file_path, encoding='utf-8') as report_json_file: + report_json = json.load(report_json_file) + test_report = TestReport().from_json(report_json) + device.add_report(test_report) + + def create_device(self, device: Device): + + # Define the device folder location + device_folder_path = os.path.join(root_dir, + LOCAL_DEVICES_DIR, + device.device_folder) + + # Create the directory + os.makedirs(device_folder_path) + + config_file_path = os.path.join(device_folder_path, + DEVICE_CONFIG) + + with open(config_file_path, 'w', encoding='utf-8') as config_file: + config_file.writelines(json.dumps(device.to_config_json(), indent=4)) + + # Ensure new folder has correct permissions + util.run_command(f"chown -R {util.get_host_user()} '{device_folder_path}'") + + # Add new device to the device repository + self._session.add_device(device) + + return device.to_config_json() + + def save_device(self, device: Device, device_json): + """Edit and save an existing device config.""" + + # Update device properties + device.manufacturer = device_json['manufacturer'] + device.model = device_json['model'] + + if 'test_modules' in device_json: + device.test_modules = device_json['test_modules'] + else: + device.test_modules = {} + + # Obtain the config file path + config_file_path = os.path.join(root_dir, + LOCAL_DEVICES_DIR, + device.device_folder, + DEVICE_CONFIG) + + with open(config_file_path, 'w+', encoding='utf-8') as config_file: + config_file.writelines(json.dumps(device.to_config_json(), indent=4)) + + return device.to_config_json() def start(self): @@ -208,7 +298,6 @@ def start(self): self.stop() def stop(self, kill=False): - self._set_status('Stopping') # Prevent discovering new devices whilst stopping if self.get_net_orc().get_listener() is not None: @@ -217,8 +306,6 @@ def stop(self, kill=False): self._stop_tests() self._stop_network(kill=kill) - self.get_session().reset() - def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) signal.signal(signal.SIGTERM, self._exit_handler) diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 7a7d19bdb..7702996c5 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -28,7 +28,7 @@ RUNTIME_DIR = "runtime/test" TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" -LOG_REGEX = r'^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_' +LOG_REGEX = r"^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_" SAVED_DEVICE_REPORTS = "local/devices/{device_folder}/reports" DEVICE_ROOT_CERTS = "local/root_certs" @@ -79,7 +79,7 @@ def run_test_modules(self): for module in self._test_modules: self._run_test_module(module) LOGGER.info("All tests complete") - + self._session.stop() self._generate_report() self._test_in_progress = False @@ -105,7 +105,7 @@ def _generate_report(self): RUNTIME_DIR, self._session.get_target_device().mac_addr.replace(":", ""), "report.json") - + with open(out_file, "w", encoding="utf-8") as f: json.dump(report, f, indent=2) util.run_command(f"chown -R {self._host_user} {out_file}") From 341f4b899041034b196c8ede1655b763467e4142 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 16 Aug 2023 09:49:03 +0100 Subject: [PATCH 2/4] Add total tests --- framework/python/src/api/api.py | 3 ++- framework/python/src/common/session.py | 26 ++++++++++++++++--- framework/python/src/test_orc/module.py | 4 +-- .../python/src/test_orc/test_orchestrator.py | 7 ++++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index c2e22d4c6..c06d9eb4c 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -115,7 +115,6 @@ async def start_test_run(self, request: Request, response: Response): return self._generate_msg(False, "Invalid request received") device = self._session.get_device(body_json["device"]["mac_addr"]) - device.firmware = body_json["device"]["firmware"] # Check Test Run is not already running if self._test_run.get_session().get_status() != "Idle": @@ -128,6 +127,8 @@ async def start_test_run(self, request: Request, response: Response): response.status_code = status.HTTP_404_NOT_FOUND return self._generate_msg(False, "A device with that MAC address could not be found") + + device.firmware = body_json["device"]["firmware"] # Check Test Run is able to start if self._test_run.get_net_orc().check_config() is False: diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 09e8ca5ed..8e3291a6f 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -38,7 +38,7 @@ def __init__(self, config_file): self._results = [] self._runtime_params = [] self._device_repository = [] - + self._total_tests = 0 self._config_file = config_file self._config = self._get_default_config() self._load_config() @@ -164,6 +164,12 @@ def set_status(self, status): def get_test_results(self): return self._results + + def get_report_tests(self): + return { + 'total': self.get_total_tests(), + 'results': self.get_test_results() + } def add_test_result(self, test_result): self._results.append(test_result) @@ -178,11 +184,20 @@ def get_all_reports(self): reports.append(device_report.to_json()) return reports + + def add_total_tests(self, no_tests): + self._total_tests += no_tests + + def get_total_tests(self): + return self._total_tests def reset(self): self.set_status('Idle') self.set_target_device(None) - self._results = [] + self._tests = { + 'total': 0, + 'results': [] + } self._started = None self._finished = None @@ -190,12 +205,17 @@ def to_json(self): # TODO: Add report URL + results = { + 'total': self.get_total_tests(), + 'results': self.get_test_results() + } + session_json = { 'status': self.get_status(), 'device': self.get_target_device(), 'started': self.get_started(), 'finished': self.get_finished(), - 'results': self.get_test_results() + 'tests': results } return session_json diff --git a/framework/python/src/test_orc/module.py b/framework/python/src/test_orc/module.py index 185940dd8..27dcfa8da 100644 --- a/framework/python/src/test_orc/module.py +++ b/framework/python/src/test_orc/module.py @@ -31,12 +31,12 @@ class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-at image_name: str = None enable_container: bool = True network: bool = True - + total_tests: int = 0 timeout: int = 60 # Absolute path dir: str = None dir_name: str = None - #Set IP Index for all test modules + # Set IP Index for all test modules ip_index: str = 9 diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 7702996c5..caa2bf3b0 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -99,7 +99,7 @@ def _generate_report(self): report["started"] = self._session.get_started().strftime("%Y-%m-%d %H:%M:%S") report["finished"] = self._session.get_finished().strftime("%Y-%m-%d %H:%M:%S") report["status"] = self._session.get_status() - report["results"] = self._session.get_test_results() + report["tests"] = self._session.get_report_tests() out_file = os.path.join( self._root_path, RUNTIME_DIR, @@ -306,6 +306,8 @@ def _run_test_module(self, module): LOGGER.error(f"Error occured whilst obbtaining results for module {module.name}") LOGGER.debug(results_error) + self._session.add_total_tests(module.total_tests) + LOGGER.info("Test module " + module.name + " has finished") def _get_module_status(self, module): @@ -371,6 +373,9 @@ def _load_test_module(self, module_dir): module.container_name = "tr-ct-" + module.dir_name + "-test" module.image_name = "test-run/" + module.dir_name + "-test" + if "tests" in module_json["config"]: + module.total_tests = len(module_json["config"]["tests"]) + if "timeout" in module_json["config"]["docker"]: module.timeout = module_json["config"]["docker"]["timeout"] From cab3c4044380e68767a3537ea70d98c9925e6af9 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 16 Aug 2023 17:39:27 +0100 Subject: [PATCH 3/4] Add report to device --- framework/python/src/api/api.py | 3 ++- framework/python/src/common/device.py | 2 +- framework/python/src/common/session.py | 8 +++++--- framework/python/src/core/testrun.py | 15 +++++++++------ .../python/src/test_orc/test_orchestrator.py | 10 ++++++---- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index c06d9eb4c..61801f203 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -127,7 +127,7 @@ async def start_test_run(self, request: Request, response: Response): response.status_code = status.HTTP_404_NOT_FOUND return self._generate_msg(False, "A device with that MAC address could not be found") - + device.firmware = body_json["device"]["firmware"] # Check Test Run is able to start @@ -135,6 +135,7 @@ async def start_test_run(self, request: Request, response: Response): response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR return self._generate_msg(False,"Configured interfaces are not ready for use. Ensure required interfaces are connected.") + self._test_run.get_session().reset() self._test_run.get_session().set_target_device(device) LOGGER.info(f"Starting Test Run with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}") diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index 2d9377a8b..276e47086 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -51,7 +51,7 @@ def to_json(self): return device_json def to_config_json(self): - """Returns the device as a python disctionary. Fields relevant to the device + """Returns the device as a python dictionary. Fields relevant to the device config json file are exported.""" device_json = {} device_json['mac_addr'] = self.mac_addr diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 8e3291a6f..fe3d47d99 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -44,7 +44,6 @@ def __init__(self, config_file): self._load_config() def start(self): - self.reset() self._status = 'Waiting for device' self._started = datetime.datetime.now() @@ -150,6 +149,9 @@ def get_device_repository(self): def add_device(self, device): self._device_repository.append(device) + def clear_device_repository(self): + self._device_repository = [] + def get_device(self, mac_addr): for device in self._device_repository: if device.mac_addr == mac_addr: @@ -164,7 +166,7 @@ def set_status(self, status): def get_test_results(self): return self._results - + def get_report_tests(self): return { 'total': self.get_total_tests(), @@ -184,7 +186,7 @@ def get_all_reports(self): reports.append(device_report.to_json()) return reports - + def add_total_tests(self, no_tests): self._total_tests += no_tests diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 7b89f46b0..7ac89b1c7 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -90,7 +90,7 @@ def __init__(self, if net_only: self._session.add_runtime_param('net_only') - self._load_all_devices() + self.load_all_devices() self._net_orc = net_orc.NetworkOrchestrator( session=self._session, @@ -101,6 +101,7 @@ def __init__(self, self._net_orc) if self._no_ui: + # Check Test Run is able to start if self.get_net_orc().check_config() is False: return @@ -108,6 +109,7 @@ def __init__(self, # Any additional checks that need to be performed go here self.start() + else: # Build UI image @@ -115,11 +117,12 @@ def __init__(self, self._api.start() # Start UI container - # Hold until API ends - while True: - time.sleep(1) + # Hold until API ends + while True: + time.sleep(1) - def _load_all_devices(self): + def load_all_devices(self): + self._session.clear_device_repository() self._load_devices(device_dir=LOCAL_DEVICES_DIR) # Temporarily removing loading of template device @@ -368,7 +371,7 @@ def _device_discovered(self, mac_addr): self.get_session().set_target_device(device) LOGGER.info( - f'Discovered {device.manufacturer} {device.model} on the network') + f'Discovered {device.manufacturer} {device.model} on the network. Waiting for device to obtain IP') def _device_stable(self, mac_addr): LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index caa2bf3b0..c62cb4262 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -21,6 +21,7 @@ from datetime import datetime from docker.types import Mount from common import logger, util +from common.testreport import TestReport from test_orc.module import TestModule LOG_NAME = "test_orc" @@ -81,19 +82,20 @@ def run_test_modules(self): LOGGER.info("All tests complete") self._session.stop() - self._generate_report() + report = TestReport().from_json(self._generate_report()) + device.add_report(report) + self._test_in_progress = False self._timestamp_results(device) + LOGGER.debug("Cleaning old test results...") self._cleanup_old_test_results(device) + LOGGER.debug("Old test results cleaned") self._test_in_progress = False def _generate_report(self): - # TODO: Calculate the status result - # We need to know the required result of each test - report = {} report["device"] = self._session.get_target_device().to_json() report["started"] = self._session.get_started().strftime("%Y-%m-%d %H:%M:%S") From 10775b49db1035bb2a77a1b608c2300fec9a8a60 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 16 Aug 2023 17:55:23 +0100 Subject: [PATCH 4/4] Only run tests if baseline passes --- .github/workflows/testing.yml | 2 ++ testing/tests/test_tests | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9e6f35323..87c8a814a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,6 +21,7 @@ jobs: testrun_tests: name: Tests runs-on: ubuntu-20.04 + needs: testrun_baseline timeout-minutes: 40 steps: - name: Checkout source @@ -28,6 +29,7 @@ jobs: - name: Run tests shell: bash {0} run: testing/tests/test_tests + pylint: name: Pylint runs-on: ubuntu-22.04 diff --git a/testing/tests/test_tests b/testing/tests/test_tests index be7a3cef3..49c77d4e4 100755 --- a/testing/tests/test_tests +++ b/testing/tests/test_tests @@ -65,7 +65,7 @@ for tester in $TESTERS; do args=$(jq -r .$tester.args $MATRIX) touch $testrun_log - sudo timeout 900 bin/testrun --single-intf --no-ui > $testrun_log 2>&1 & + sudo timeout 900 bin/testrun --single-intf --no-ui --no-validate > $testrun_log 2>&1 & TPID=$! # Time to wait for testrun to be ready