From 99644b72270ea7a993c994b09246dd03a4931cf6 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Tue, 6 Aug 2024 14:48:39 -0600 Subject: [PATCH 1/9] Update NTP report (#666) * Update NTP report * cleanup imports * pylint updates --- modules/test/ntp/python/src/ntp_module.py | 76 +- .../unit/ntp/reports/ntp_report_local.html | 1398 +---------------- 2 files changed, 74 insertions(+), 1400 deletions(-) diff --git a/modules/test/ntp/python/src/ntp_module.py b/modules/test/ntp/python/src/ntp_module.py index 033e98974..be27abbad 100644 --- a/modules/test/ntp/python/src/ntp_module.py +++ b/modules/test/ntp/python/src/ntp_module.py @@ -14,8 +14,8 @@ """NTP test module""" from test_module import TestModule from scapy.all import rdpcap, IP, IPv6, NTP, UDP, Ether -from datetime import datetime import os +from collections import defaultdict LOG_NAME = 'test_ntp' MODULE_REPORT_FILE_NAME = 'ntp_report.html' @@ -69,6 +69,33 @@ def generate_module_report(self): total_responses = sum(1 for row in ntp_table_data if row['Type'] == 'Server') + # Initialize a dictionary to store timestamps for each unique combination + timestamps = defaultdict(list) + + # Collect timestamps for each unique combination + for row in ntp_table_data: + # Add the timestamp to the corresponding combination + key = (row['Source'], row['Destination'], row['Type'], row['Version']) + timestamps[key].append(row['Timestamp']) + + # Calculate the average time between requests for each unique combination + average_time_between_requests = {} + + for key, times in timestamps.items(): + # Sort the timestamps + times.sort() + + # Calculate the time differences between consecutive timestamps + time_diffs = [t2 - t1 for t1, t2 in zip(times[:-1], times[1:])] + + # Calculate the average of the time differences + if time_diffs: + avg_diff = sum(time_diffs) / len(time_diffs) + else: + avg_diff = 0 # one timestamp, the average difference is 0 + + average_time_between_requests[key] = avg_diff + # Add summary table html_content += (f''' @@ -92,7 +119,6 @@ def generate_module_report(self): ''') if total_requests + total_responses > 0: - table_content = '''
@@ -101,37 +127,39 @@ def generate_module_report(self): - + + ''' - for row in ntp_table_data: - - # Timestamp of the NTP packet - dt_object = datetime.utcfromtimestamp(row['Timestamp']) + # Generate the HTML table with the count column + for (src, dst, typ, + version), avg_diff in average_time_between_requests.items(): + cnt = len(timestamps[(src, dst, typ, version)]) - # Extract milliseconds from the fractional part of the timestamp - milliseconds = int((row['Timestamp'] % 1) * 1000) - - # Format the datetime object with milliseconds - formatted_time = dt_object.strftime( - '%b %d, %Y %H:%M:%S.') + f'{milliseconds:03d}' + # Sync Average only applies to client requests + if 'Client' in typ: + # Convert avg_diff to seconds and format it + avg_diff_seconds = avg_diff + avg_formatted_time = f'{avg_diff_seconds:.3f} seconds' + else: + avg_formatted_time = 'N/A' - table_content += (f''' + table_content += f''' - - - - - - ''') + + + + + + + ''' table_content += '''
Destination Type VersionTimestampCountSync Request Average
{row['Source']}{row['Destination']}{row['Type']}{row['Version']}{formatted_time}
{src}{dst}{typ}{version}{cnt}{avg_formatted_time}
''' - html_content += table_content else: @@ -159,8 +187,8 @@ def extract_ntp_data(self): # Read the pcap files packets = (rdpcap(self.startup_capture_file) + - rdpcap(self.monitor_capture_file) + - rdpcap(self.ntp_server_capture_file)) + rdpcap(self.monitor_capture_file) + + rdpcap(self.ntp_server_capture_file)) # Iterate through NTP packets for packet in packets: @@ -283,7 +311,7 @@ def _ntp_network_ntp_dhcp(self): 'server and non-DHCP provided server') elif ntp_to_remote: result = ('Feature Not Detected', - 'Device sent NTP request to non-DHCP provided server') + 'Device sent NTP request to non-DHCP provided server') elif ntp_to_local: result = True, 'Device sent NTP request to DHCP provided server' diff --git a/testing/unit/ntp/reports/ntp_report_local.html b/testing/unit/ntp/reports/ntp_report_local.html index a08c42f9d..c9715fba5 100644 --- a/testing/unit/ntp/reports/ntp_report_local.html +++ b/testing/unit/ntp/reports/ntp_report_local.html @@ -25,7 +25,8 @@

NTP Module

Destination Type Version - Timestamp + Count + Sync Request Average @@ -34,1435 +35,80 @@

NTP Module

216.239.35.12 Client 4 - Feb 15, 2024 22:12:28.681 + 8 + 37.942 seconds 216.239.35.12 10.10.10.15 Server 4 - Feb 15, 2024 22:12:28.728 + 8 + N/A 10.10.10.15 216.239.35.4 Client 4 - Feb 15, 2024 22:12:28.842 + 8 + 37.834 seconds 216.239.35.4 10.10.10.15 Server 4 - Feb 15, 2024 22:12:28.888 + 8 + N/A 10.10.10.15 216.239.35.8 Client 4 - Feb 15, 2024 22:12:29.042 + 8 + 38.056 seconds 216.239.35.8 10.10.10.15 Server 4 - Feb 15, 2024 22:12:29.089 + 8 + N/A 10.10.10.15 216.239.35.0 Client 4 - Feb 15, 2024 22:12:29.243 + 14 + 20.601 seconds 216.239.35.0 10.10.10.15 Server 4 - Feb 15, 2024 22:12:29.290 + 17 + N/A 10.10.10.15 10.10.10.5 Client 4 - Feb 15, 2024 22:12:29.447 + 63 + 13.057 seconds 10.10.10.5 10.10.10.15 Server 4 - Feb 15, 2024 22:12:29.448 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:12:30.802 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:30.850 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:12:30.973 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.032 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:12:31.173 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.220 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:12:31.376 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.423 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:12:32.867 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:32.914 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:12:33.112 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.159 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:12:33.271 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.318 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:12:33.475 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.522 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:12:34.956 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.002 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:12:35.182 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.228 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:12:35.398 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.445 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:12:35.625 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.673 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:35.785 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.786 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:41.931 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:41.932 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:43.954 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:43.956 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:06.439 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:06.489 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:08.492 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:08.494 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:08.543 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:13:40.310 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.357 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:13:40.512 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:40.536 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.542 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.574 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.583 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:13:40.714 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.764 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:13:40.917 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.965 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:48.274 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:48.277 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:12.619 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:12.624 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:12.668 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:14:44.515 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:44.562 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:44.702 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:44.704 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:14:45.158 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:45.219 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:14:45.359 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:45.406 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:14:45.707 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:45.755 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:14:45.980 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:46.027 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:53.026 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:53.029 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:16.786 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:16.791 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:15:18.794 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:18.843 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:48.884 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:48.887 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:15:49.063 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:49.110 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:15:49.462 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:49.509 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:15:50.127 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:50.175 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:15:51.107 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:51.154 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:15:51.890 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:51.938 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:20.970 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:20.971 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:16:24.975 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:25.023 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:16:53.677 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:53.739 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:16:54.276 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.322 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:16:54.593 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.648 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:16:55.435 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:55.481 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:16:57.059 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:57.107 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:02.738 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:02.740 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:26.136 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:26.139 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:29.447 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:29.448 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:35.785 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.786 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:41.931 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:41.932 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:43.954 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:43.956 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:08.492 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:08.494 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:40.536 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.541 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:48.274 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:48.277 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:12.619 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:12.624 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:44.702 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:44.703 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:53.026 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:53.029 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:16.786 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:16.791 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:48.884 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:48.887 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:20.970 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:20.970 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:02.738 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:02.740 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:26.136 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:26.139 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:59.293 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:59.293 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:18:07.242 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:18:07.242 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:18:32.379 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:18:32.379 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:06.908 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:06.908 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:08.936 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:08.937 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:10.974 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:10.974 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:12.998 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:12.999 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:59.581 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:59.582 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:34.063 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:34.063 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:36.121 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:36.121 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:38.176 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:38.176 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:40.277 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:40.277 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:22:05.704 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:22:05.706 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:22:45.469 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:22:45.470 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:23:09.826 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:23:09.828 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:23:50.337 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:23:50.343 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:24:13.945 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:24:13.946 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:24:54.876 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:24:54.877 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:25:59.000 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:25:59.001 + 63 + N/A From 597bb55dd94d5273a9e33b68a07352f805fc78c0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Nikitsin Date: Mon, 12 Aug 2024 18:45:06 +0200 Subject: [PATCH 2/9] testrun statuses class --- framework/python/src/api/api.py | 25 +++++++++++--- framework/python/src/common/config.py | 34 +++++++++++++++++++ framework/python/src/common/session.py | 11 +++--- framework/python/src/core/testrun.py | 5 +-- .../src/net_orc/network_orchestrator.py | 15 ++++---- .../python/src/test_orc/test_orchestrator.py | 20 ++++++----- 6 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 framework/python/src/common/config.py diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 719710694..3e3a89743 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -28,6 +28,7 @@ from common import logger, tasks from common.device import Device +from common.config import TestrunStatuses LOGGER = logger.get_logger("api") @@ -204,7 +205,9 @@ async def start_test_run(self, request: Request, response: Response): # Check Testrun is not already running if self._test_run.get_session().get_status() in [ - "In Progress", "Waiting for Device", "Monitoring" + TestrunStatuses.IN_PROGRESS, + TestrunStatuses.WAITING_FOR_DEVICE, + TestrunStatuses.MONITORING ]: LOGGER.debug("Testrun is already running. Cannot start another instance") response.status_code = status.HTTP_409_CONFLICT @@ -269,7 +272,9 @@ async def stop_test_run(self, response: Response): # Check if Testrun is running if (self._test_run.get_session().get_status() - not in ["In Progress", "Waiting for Device", "Monitoring"]): + not in [TestrunStatuses.IN_PROGRESS, + TestrunStatuses.WAITING_FOR_DEVICE, + TestrunStatuses.MONITORING]): response.status_code = 404 return self._generate_msg(False, "Testrun is not currently running") @@ -286,7 +291,11 @@ def shutdown(self, response: Response): # Check that Testrun is not currently running if (self._session.get_status() - not in ["Cancelled", "Compliant", "Non-Compliant", "Idle"]): + not in [TestrunStatuses.CANCELLED, + TestrunStatuses.COMPLIANT, + TestrunStatuses.NON_COMPLIANT, + TestrunStatuses.IDLE + ]): LOGGER.debug("Unable to shutdown Testrun as Testrun is in progress") response.status_code = 400 return self._generate_msg( @@ -430,7 +439,10 @@ async def delete_device(self, request: Request, response: Response): # Check that Testrun is not currently running against this device if (self._session.get_target_device() == device and self._session.get_status() - not in ["Cancelled", "Compliant", "Non-Compliant"]): + not in [TestrunStatuses.CANCELLED, + TestrunStatuses.COMPLIANT, + TestrunStatuses.NON_COMPLIANT + ]): response.status_code = 403 return self._generate_msg( False, "Cannot delete this device whilst " + "it is being tested") @@ -532,7 +544,10 @@ async def edit_device(self, request: Request, response: Response): if (self._session.get_target_device() == device and self._session.get_status() - not in ["Cancelled", "Compliant", "Non-Compliant"]): + not in [TestrunStatuses.CANCELLED, + TestrunStatuses.COMPLIANT, + TestrunStatuses.NON_COMPLIANT + ]): response.status_code = 403 return self._generate_msg( False, "Cannot edit this device whilst " + "it is being tested") diff --git a/framework/python/src/common/config.py b/framework/python/src/common/config.py new file mode 100644 index 000000000..06e25a09c --- /dev/null +++ b/framework/python/src/common/config.py @@ -0,0 +1,34 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testrun config""" + + +class TestrunStatuses: + IDLE = "Idle" + WAITING_FOR_DEVICE = "Waiting for Device" + MONITORING = "Monitoring" + IN_PROGRESS = "In Progress" + CANCELLED = "Cancelled" + COMPLIANT = "Compliant" + NON_COMPLIANT = "Non-Compliant" + STOPPING = "Stopping" + +class TestResults: + IN_PROGRESS = "In Progress" + COMPLIANT = "Compliant" + NON_COMPLIANT = "Non-Compliant" + ERROR = "Error" + FEATURE_NOT_DETECTED = "Feature Not Detected" + + diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index d80ad023b..b5eca5271 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -20,6 +20,7 @@ from fastapi.encoders import jsonable_encoder from common import util, logger, mqtt from common.risk_profile import RiskProfile +from common.config import TestrunStatuses from net_orc.ip_control import IPControl # Certificate dependencies @@ -52,7 +53,7 @@ def wrapper(self, *args, **kwargs): result = method(self, *args, **kwargs) - if self.get_status() != 'Idle': + if self.get_status() != TestrunStatuses.IDLE: self.get_mqtt_client().send_message( STATUS_TOPIC, jsonable_encoder(self.to_json()) @@ -80,7 +81,7 @@ class TestrunSession(): def __init__(self, root_dir): self._root_dir = root_dir - self._status = 'Idle' + self._status = TestrunStatuses.IDLE # Target test device self._device = None @@ -144,7 +145,7 @@ def __init__(self, root_dir): def start(self): self.reset() - self._status = 'Waiting for Device' + self._status = TestrunStatuses.WAITING_FOR_DEVICE self._started = datetime.datetime.now() def get_started(self): @@ -288,7 +289,7 @@ def set_config(self, config_json): self._save_config() # Update log level - LOGGER.debug(f'Setting log level to {config_json["log_level"]}') + LOGGER.debug(f'Setting log level to {config_json["log_level"]}') # pylint: disable=W1405 logger.set_log_level(config_json['log_level']) def set_target_device(self, device): @@ -592,7 +593,7 @@ def delete_profile(self, profile): return False def reset(self): - self.set_status('Idle') + self.set_status(TestrunStatuses.IDLE) self.set_target_device(None) self._report_url = None self._total_tests = 0 diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 6f6baf642..5723db1a4 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -31,6 +31,7 @@ from common.device import Device from common.session import TestrunSession from common.testreport import TestReport +from common.config import TestrunStatuses from api.api import Api from net_orc.listener import NetworkEvent from net_orc import network_orchestrator as net_orc @@ -372,7 +373,7 @@ def stop(self): self._stop_tests() self._stop_network(kill=True) - self.get_session().set_status('Cancelled') + self.get_session().set_status(TestrunStatuses.CANCELLED) def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -454,7 +455,7 @@ def _device_discovered(self, mac_addr): def _device_stable(self, mac_addr): # Do not continue testing if Testrun has cancelled during monitor phase - if self.get_session().get_status() == 'Cancelled': + if self.get_session().get_status() == TestrunStatuses.CANCELLED: self._stop_network() return diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index eef418d52..06c7788d7 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -25,6 +25,7 @@ import traceback from docker.types import Mount from common import logger, util, mqtt +from common.config import TestrunStatuses from net_orc.listener import Listener from net_orc.network_event import NetworkEvent from net_orc.network_validator import NetworkValidator @@ -216,7 +217,7 @@ def _device_discovered(self, mac_addr): if device.ip_addr is None: LOGGER.info( f'Timed out whilst waiting for {mac_addr} to obtain an IP address') - self._session.set_status('Cancelled') + self._session.set_status(TestrunStatuses.CANCELLED) return LOGGER.info( f'Device with mac addr {device.mac_addr} has obtained IP address ' @@ -276,7 +277,7 @@ def _dhcp_lease_ack(self, packet): def _start_device_monitor(self, device): """Start a timer until the steady state has been reached and callback the steady state method for this device.""" - self.get_session().set_status('Monitoring') + self.get_session().set_status(TestrunStatuses.MONITORING) self._monitor_packets = [] LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' f'for {str(self._session.get_monitor_period())} seconds') @@ -293,14 +294,14 @@ def _start_device_monitor(self, device): time.sleep(1) # Check Testrun hasn't been cancelled - if self._session.get_status() == 'Cancelled': + if self._session.get_status() == TestrunStatuses.CANCELLED: sniffer.stop() return if not self._ip_ctrl.check_interface_status( self._session.get_device_interface()): sniffer.stop() - self._session.set_status('Cancelled') + self._session.set_status(TestrunStatuses.CANCELLED) LOGGER.error('Device interface disconnected, cancelling Testrun') LOGGER.debug('Writing packets to monitor.pcap') @@ -810,8 +811,10 @@ def internet_conn_checker(self, mqtt_client: mqtt.MQTT, topic: str): # Only check if Testrun is running if self.get_session().get_status() not in [ - 'Waiting for Device', 'Monitoring', 'In Progress' - ]: + TestrunStatuses.WAITING_FOR_DEVICE, + TestrunStatuses.MONITORING, + TestrunStatuses.IN_PROGRESS + ]: message['connection'] = None # Only run if single intf mode not used diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index a38371d07..b2b48a6dc 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -23,6 +23,7 @@ from docker.types import Mount from common import logger, util from common.testreport import TestReport +from common.config import TestrunStatuses from test_orc.module import TestModule from test_orc.test_case import TestCase import threading @@ -84,7 +85,7 @@ def run_test_modules(self): """Iterates through each test module and starts the container.""" # Do not start test modules if status is not in progress, e.g. Stopping - if self.get_session().get_status() != "In Progress": + if self.get_session().get_status() != TestrunStatuses.IN_PROGRESS: return device = self._session.get_target_device() @@ -118,8 +119,8 @@ def run_test_modules(self): self._session.finish() # Do not carry on (generating a report) if Testrun has been stopped - if self.get_session().get_status() != "In Progress": - return "Cancelled" + if self.get_session().get_status() != TestrunStatuses.IN_PROGRESS: + return TestrunStatuses.CANCELLED report = TestReport() report.from_json(self._generate_report()) @@ -328,9 +329,9 @@ def zip_results(self, # Check that the ZIP was successfully created zip_file = zip_location + ".zip" - LOGGER.info(f'''Archive {'created at ' + zip_file + LOGGER.info(f"""Archive {"created at " + zip_file if os.path.exists(zip_file) - else'creation failed'}''') + else "creation failed"}""") return zip_file @@ -362,7 +363,7 @@ def _run_test_module(self, module): """Start the test container and extract the results.""" # Check that Testrun is not stopping - if self.get_session().get_status() != "In Progress": + if self.get_session().get_status() != TestrunStatuses.IN_PROGRESS: return device = self._session.get_target_device() @@ -376,7 +377,7 @@ def _run_test_module(self, module): if not self._net_orc.is_device_connected(): LOGGER.error("Device was disconnected") self._set_test_modules_error(current_test) - self._session.set_status("Cancelled") + self._session.set_status(TestrunStatuses.CANCELLED) return test_copy = copy.deepcopy(test) @@ -475,7 +476,8 @@ def _run_test_module(self, module): log_thread.daemon = True log_thread.start() - while (status == "running" and self._session.get_status() == "In Progress"): + while (status == "running" + and self._session.get_status() == TestrunStatuses.IN_PROGRESS): if time.time() > test_module_timeout: LOGGER.error("Module timeout exceeded, killing module: " + module.name) self._stop_module(module=module, kill=True) @@ -488,7 +490,7 @@ def _run_test_module(self, module): f.write(line + "\n") # Check that Testrun has not been stopped whilst this module was running - if self.get_session().get_status() == "Stopping": + if self.get_session().get_status() == TestrunStatuses.STOPPING: # Discard results for this module LOGGER.info(f"Test module {module.name} has forcefully quit") return From 4fad42cd300b82ffaa0d8c2aef11d3cedd50bf1f Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 13 Aug 2024 15:48:15 +0100 Subject: [PATCH 3/9] Add not started and disabled as test statuses --- framework/python/src/common/session.py | 1 - .../python/src/test_orc/test_orchestrator.py | 17 +++++++++++++++++ make/DEBIAN/control | 2 +- modules/test/base/python/src/test_module.py | 5 ++--- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 02d46591e..7c2ef5e8f 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -331,7 +331,6 @@ def add_test_result(self, result): updated = True if not updated: - result.result = 'In Progress' self._results.append(result) def add_module_report(self, module_report): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 13b466337..aeae604ae 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -100,6 +100,23 @@ def run_test_modules(self): continue test_modules.append(module) + + for test in module.tests: + + # Duplicate test obj so we don't alter the source + test_copy = copy.deepcopy(test) + + # Set result to Not Started + test_copy.result = "Not Started" + + # We don't want steps to resolve for not started tests + if hasattr(test_copy, "recommendations"): + test_copy.recommendations = None + + # Add test result to the session + self.get_session().add_test_result(test_copy) + + # Increment number of tests that will be run self.get_session().add_total_tests(len(module.tests)) for module in test_modules: diff --git a/make/DEBIAN/control b/make/DEBIAN/control index cecad9d17..3361b7ff1 100644 --- a/make/DEBIAN/control +++ b/make/DEBIAN/control @@ -1,5 +1,5 @@ Package: Testrun -Version: 1.4-a +Version: 1.4.1-a Architecture: amd64 Maintainer: Google Homepage: https://github.com/google/testrun diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index deed0d978..9c06c72a9 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -121,9 +121,8 @@ def run_tests(self): test['description'] = 'This test could not be found' else: LOGGER.debug(f'Test {test["name"]} is disabled') - - # To be added in v1.3.2 - # result = 'Disabled', 'This test is disabled and did not run' + test['result'] = 'Disabled' + test['description'] = 'This test did not run because it is disabled' if result is not None: # Compliant or non-compliant as a boolean only From 3d7dc2761fad8293595d540dd0dc308d5643a3d3 Mon Sep 17 00:00:00 2001 From: Aliaksandr Nikitsin Date: Tue, 13 Aug 2024 15:27:44 +0000 Subject: [PATCH 4/9] test results --- framework/python/src/common/config.py | 1 + framework/python/src/common/session.py | 8 ++++---- framework/python/src/common/testreport.py | 17 +++++++++-------- framework/python/src/core/testrun.py | 2 +- framework/python/src/test_orc/test_case.py | 3 ++- .../python/src/test_orc/test_orchestrator.py | 16 ++++++++-------- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/framework/python/src/common/config.py b/framework/python/src/common/config.py index 06e25a09c..7d52de4ac 100644 --- a/framework/python/src/common/config.py +++ b/framework/python/src/common/config.py @@ -30,5 +30,6 @@ class TestResults: NON_COMPLIANT = "Non-Compliant" ERROR = "Error" FEATURE_NOT_DETECTED = "Feature Not Detected" + INFORMATIONAL = "Informational" diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index b5eca5271..2c37850c8 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -20,7 +20,7 @@ from fastapi.encoders import jsonable_encoder from common import util, logger, mqtt from common.risk_profile import RiskProfile -from common.config import TestrunStatuses +from common.config import TestrunStatuses, TestResults from net_orc.ip_control import IPControl # Certificate dependencies @@ -161,8 +161,8 @@ def stop(self): def finish(self): # Set any in progress test results to Error for test_result in self._results: - if test_result.result == 'In Progress': - test_result.result = 'Error' + if test_result.result == TestResults.IN_PROGRESS: + test_result.result = TestResults.ERROR self._finished = datetime.datetime.now() @@ -370,7 +370,7 @@ def add_test_result(self, result): def set_test_result_error(self, result): """Set test result error""" - result.result = 'Error' + result.result = TestResults.ERROR self._results.append(result) def add_module_report(self, module_report): diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py index 88a25a2b1..0c75cb90f 100644 --- a/framework/python/src/common/testreport.py +++ b/framework/python/src/common/testreport.py @@ -17,6 +17,7 @@ from weasyprint import HTML from io import BytesIO from common import util +from common.config import TestResults, TestrunStatuses import base64 import os from test_orc.test_case import TestCase @@ -43,7 +44,7 @@ class TestReport(): """Represents a previous Testrun report.""" def __init__(self, - status='Non-Compliant', + status=TestrunStatuses.NON_COMPLIANT, started=None, finished=None, total_tests=0): @@ -430,7 +431,7 @@ def generate_results(self, json_data, page_num): successful_tests = 0 for test in json_data['tests']['results']: - if test['result'] != 'Error': + if test['result'] != TestResults.ERROR: successful_tests += 1 result_list = f''' @@ -457,15 +458,15 @@ def generate_results(self, json_data, page_num): return result_list def generate_result(self, result): - if result['result'] == 'Non-Compliant': + if result['result'] == TestResults.NON_COMPLIANT: result_class = 'result-test-result-non-compliant' - elif result['result'] == 'Compliant': + elif result['result'] == TestResults.COMPLIANT: result_class = 'result-test-result-compliant' - elif result['result'] == 'Error': + elif result['result'] == TestResults.ERROR: result_class = 'result-test-result-error' - elif result['result'] == 'Feature Not Detected': + elif result['result'] == TestResults.FEATURE_NOT_DETECTED: result_class = 'result-test-result-feature-not-detected' - elif result['result'] == 'Informational': + elif result['result'] == TestResults.INFORMATIONAL: result_class = 'result-test-result-informational' else: result_class = 'result-test-result-skipped' @@ -615,7 +616,7 @@ def generate_device_module_label(self, module, enabled): return label def generate_result_summary(self, json_data): - if json_data['status'] == 'Compliant': + if json_data['status'] == TestResults.COMPLIANT: result_summary = '''
''' else: diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 5723db1a4..1c8b56704 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -460,7 +460,7 @@ def _device_stable(self, mac_addr): return LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') - self._set_status('In Progress') + self._set_status(TestrunStatuses.IN_PROGRESS) result = self._test_orc.run_test_modules() if result is not None: diff --git a/framework/python/src/test_orc/test_case.py b/framework/python/src/test_orc/test_case.py index cf0d6593a..750886d61 100644 --- a/framework/python/src/test_orc/test_case.py +++ b/framework/python/src/test_orc/test_case.py @@ -14,6 +14,7 @@ """Represents an individual test case.""" from dataclasses import dataclass, field +from common.config import TestResults @dataclass @@ -24,7 +25,7 @@ class TestCase: # pylint: disable=too-few-public-methods,too-many-instance-attr description: str = "" expected_behavior: str = "" required_result: str = "Recommended" - result: str = "Non-Compliant" + result: str = TestResults.NON_COMPLIANT recommendations: list = field(default_factory=lambda: []) def to_dict(self): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index b2b48a6dc..62cdb5e83 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -23,7 +23,7 @@ from docker.types import Mount from common import logger, util from common.testreport import TestReport -from common.config import TestrunStatuses +from common.config import TestrunStatuses, TestResults from test_orc.module import TestModule from test_orc.test_case import TestCase import threading @@ -187,16 +187,16 @@ def _generate_report(self): return report def _calculate_result(self): - result = "Compliant" + result = TestResults.COMPLIANT for test_result in self._session.get_test_results(): # Check Required tests if (test_result.required_result.lower() == "required" and test_result.result.lower() != "compliant"): - result = "Non-Compliant" + result = TestResults.NON_COMPLIANT # Check Required if Applicable tests elif (test_result.required_result.lower() == "required if applicable" and test_result.result.lower() == "non-compliant"): - result = "Non-Compliant" + result = TestResults.NON_COMPLIANT return result def _cleanup_old_test_results(self, device): @@ -381,7 +381,7 @@ def _run_test_module(self, module): return test_copy = copy.deepcopy(test) - test_copy.result = "In Progress" + test_copy.result = TestResults.IN_PROGRESS # We don't want steps to resolve for in progress tests if hasattr(test_copy, "recommendations"): @@ -518,11 +518,11 @@ def _run_test_module(self, module): result=test_result["result"]) # Any informational test should always report informational - if test_case.required_result == "Informational": - test_case.result = "Informational" + if test_case.required_result == TestResults.INFORMATIONAL: + test_case.result = TestResults.INFORMATIONAL # Add steps to resolve if test is non-compliant - if (test_case.result == "Non-Compliant" and + if (test_case.result == TestResults.NON_COMPLIANT and "recommendations" in test_result): test_case.recommendations = test_result["recommendations"] else: From 991a49d47b98e5311e3f4d014dd5c99303244586 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 13 Aug 2024 16:44:12 +0100 Subject: [PATCH 5/9] Fix disabled test --- modules/test/base/python/src/test_module.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 9c06c72a9..1a3c6b908 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -116,21 +116,20 @@ def run_tests(self): LOGGER.error(e) traceback.print_exc() else: - LOGGER.info(f'Test {test["name"]} not implemented. Skipping') - test['result'] = 'Error' - test['description'] = 'This test could not be found' + LOGGER.error(f'Test {test["name"]} has not been implemented') + result = 'Error', 'This test could not be found' else: LOGGER.debug(f'Test {test["name"]} is disabled') - test['result'] = 'Disabled' - test['description'] = 'This test did not run because it is disabled' + result = 'Disabled', 'This test did not run because it is disabled' + # Check if the test module has returned a result if result is not None: + # Compliant or non-compliant as a boolean only if isinstance(result, bool): test['result'] = 'Compliant' if result else 'Non-Compliant' test['description'] = 'No description was provided for this test' else: - # TODO: This is assuming that result is an array but haven't checked # Error result if result[0] is None: test['result'] = 'Error' @@ -142,6 +141,7 @@ def run_tests(self): # Compliant / Non-Compliant result elif isinstance(result[0], bool): test['result'] = 'Compliant' if result[0] else 'Non-Compliant' + # Result may be a string, e.g Error, Feature Not Detected elif isinstance(result[0], str): test['result'] = result[0] @@ -159,6 +159,7 @@ def run_tests(self): if len(result)>2: test['details'] = result[2] else: + LOGGER.debug('No result was returned from the test module') test['result'] = 'Error' test['description'] = 'An error occured whilst running this test' From 16bd2d77011bc1a587f3e2f762de94adb8e8221b Mon Sep 17 00:00:00 2001 From: J Boddey Date: Wed, 14 Aug 2024 11:29:50 +0100 Subject: [PATCH 6/9] Check if profiles format has loaded (#665) --- framework/python/src/api/api.py | 7 +++++++ framework/python/src/common/session.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 719710694..041be4cc9 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -681,6 +681,12 @@ async def update_profile(self, request: Request, response: Response): LOGGER.debug("Received profile update request") + # Check if the profiles format was loaded correctly + if self.get_session().get_profiles_format() is None: + response.status_code = status.HTTP_501_NOT_IMPLEMENTED + return self._generate_msg(False, + "Risk profiles are not available right now") + try: req_raw = (await request.body()).decode("UTF-8") req_json = json.loads(req_raw) @@ -696,6 +702,7 @@ async def update_profile(self, request: Request, response: Response): profile = self.get_session().get_profile(profile_name) if profile is None: + # Create new profile profile = self.get_session().update_profile(req_json) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 7c2ef5e8f..a147ce6c0 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -375,6 +375,10 @@ def _load_profiles(self): 'An error occurred whilst loading the risk assessment format') LOGGER.debug(e) + # If the format JSON fails to load, skip loading profiles + LOGGER.error('Profiles will not be loaded') + return + profile_format_array = [] # Remove internal properties From 6fd797d4ef347ab2f5ff1aebe05033c84ec5d406 Mon Sep 17 00:00:00 2001 From: Aliaksandr Nikitsin Date: Wed, 14 Aug 2024 13:30:37 +0200 Subject: [PATCH 7/9] add disable and not startet to config --- framework/python/src/common/config.py | 4 ++++ framework/python/src/common/session.py | 2 +- .../python/src/test_orc/test_orchestrator.py | 2 +- modules/test/base/python/src/test_module.py | 20 +++++++++---------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/framework/python/src/common/config.py b/framework/python/src/common/config.py index 7d52de4ac..4cae24cea 100644 --- a/framework/python/src/common/config.py +++ b/framework/python/src/common/config.py @@ -24,6 +24,7 @@ class TestrunStatuses: NON_COMPLIANT = "Non-Compliant" STOPPING = "Stopping" + class TestResults: IN_PROGRESS = "In Progress" COMPLIANT = "Compliant" @@ -31,5 +32,8 @@ class TestResults: ERROR = "Error" FEATURE_NOT_DETECTED = "Feature Not Detected" INFORMATIONAL = "Informational" + NOT_STARTED = "Not Started" + DISABLED = "Disabled" + diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 8021d161e..5ed0585e6 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -155,7 +155,7 @@ def get_finished(self): return self._finished def stop(self): - self.set_status('Stopping') + self.set_status(TestrunStatuses.STOPPING) self.finish() def finish(self): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 809250381..d6954179d 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -110,7 +110,7 @@ def run_test_modules(self): test_copy = copy.deepcopy(test) # Set result to Not Started - test_copy.result = "Not Started" + test_copy.result = TestResults.NOT_STARTED # We don't want steps to resolve for not started tests if hasattr(test_copy, "recommendations"): diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 1a3c6b908..08a664a84 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -19,6 +19,8 @@ from datetime import datetime import traceback +from common.config import TestResults + LOGGER = None RESULTS_DIR = '/runtime/output/' CONF_FILE = '/testrun/conf/module_config.json' @@ -117,22 +119,22 @@ def run_tests(self): traceback.print_exc() else: LOGGER.error(f'Test {test["name"]} has not been implemented') - result = 'Error', 'This test could not be found' + result = TestResults.ERROR, 'This test could not be found' else: LOGGER.debug(f'Test {test["name"]} is disabled') - result = 'Disabled', 'This test did not run because it is disabled' + result = TestResults.DISABLED, 'This test did not run because it is disabled' # Check if the test module has returned a result if result is not None: # Compliant or non-compliant as a boolean only if isinstance(result, bool): - test['result'] = 'Compliant' if result else 'Non-Compliant' + test['result'] = TestResults.COMPLIANT if result else TestResults.NON_COMPLIANT test['description'] = 'No description was provided for this test' else: # Error result if result[0] is None: - test['result'] = 'Error' + test['result'] = TestResults.ERROR if len(result) > 1: test['description'] = result[1] else: @@ -140,14 +142,13 @@ def run_tests(self): # Compliant / Non-Compliant result elif isinstance(result[0], bool): - test['result'] = 'Compliant' if result[0] else 'Non-Compliant' - + test['result'] = TestResults.COMPLIANT if result[0] else TestResults.NON_COMPLIANT # Result may be a string, e.g Error, Feature Not Detected elif isinstance(result[0], str): test['result'] = result[0] else: LOGGER.error(f'Unknown result detected: {result[0]}') - test['result'] = 'Error' + test['result'] = TestResults.ERROR # Check that description is a string if isinstance(result[1], str): @@ -159,12 +160,11 @@ def run_tests(self): if len(result)>2: test['details'] = result[2] else: - LOGGER.debug('No result was returned from the test module') - test['result'] = 'Error' + test['result'] = TestResults.ERROR test['description'] = 'An error occured whilst running this test' # Remove the steps to resolve if compliant already - if (test['result'] == 'Compliant' and 'recommendations' in test): + if (test['result'] == TestResults.COMPLIANT and 'recommendations' in test): test.pop('recommendations') test['end'] = datetime.now().isoformat() From 9f4f7581ee2a64137bf92e42ad4133121535223d Mon Sep 17 00:00:00 2001 From: Aliaksandr Nikitsin Date: Wed, 14 Aug 2024 15:35:21 +0200 Subject: [PATCH 8/9] rename statuses --- framework/python/src/api/api.py | 34 +++++++++---------- framework/python/src/common/session.py | 18 +++++----- .../src/common/{config.py => statuses.py} | 4 +-- framework/python/src/common/testreport.py | 18 +++++----- framework/python/src/core/testrun.py | 8 ++--- .../src/net_orc/network_orchestrator.py | 16 ++++----- framework/python/src/test_orc/test_case.py | 4 +-- .../python/src/test_orc/test_orchestrator.py | 32 ++++++++--------- modules/test/base/python/src/test_module.py | 18 +++++----- 9 files changed, 76 insertions(+), 76 deletions(-) rename framework/python/src/common/{config.py => statuses.py} (96%) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 49ec29f39..248455e7e 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -28,7 +28,7 @@ from common import logger, tasks from common.device import Device -from common.config import TestrunStatuses +from common.statuses import TestrunStatus LOGGER = logger.get_logger("api") @@ -205,9 +205,9 @@ async def start_test_run(self, request: Request, response: Response): # Check Testrun is not already running if self._test_run.get_session().get_status() in [ - TestrunStatuses.IN_PROGRESS, - TestrunStatuses.WAITING_FOR_DEVICE, - TestrunStatuses.MONITORING + TestrunStatus.IN_PROGRESS, + TestrunStatus.WAITING_FOR_DEVICE, + TestrunStatus.MONITORING ]: LOGGER.debug("Testrun is already running. Cannot start another instance") response.status_code = status.HTTP_409_CONFLICT @@ -272,9 +272,9 @@ async def stop_test_run(self, response: Response): # Check if Testrun is running if (self._test_run.get_session().get_status() - not in [TestrunStatuses.IN_PROGRESS, - TestrunStatuses.WAITING_FOR_DEVICE, - TestrunStatuses.MONITORING]): + not in [TestrunStatus.IN_PROGRESS, + TestrunStatus.WAITING_FOR_DEVICE, + TestrunStatus.MONITORING]): response.status_code = 404 return self._generate_msg(False, "Testrun is not currently running") @@ -291,10 +291,10 @@ def shutdown(self, response: Response): # Check that Testrun is not currently running if (self._session.get_status() - not in [TestrunStatuses.CANCELLED, - TestrunStatuses.COMPLIANT, - TestrunStatuses.NON_COMPLIANT, - TestrunStatuses.IDLE + not in [TestrunStatus.CANCELLED, + TestrunStatus.COMPLIANT, + TestrunStatus.NON_COMPLIANT, + TestrunStatus.IDLE ]): LOGGER.debug("Unable to shutdown Testrun as Testrun is in progress") response.status_code = 400 @@ -439,9 +439,9 @@ async def delete_device(self, request: Request, response: Response): # Check that Testrun is not currently running against this device if (self._session.get_target_device() == device and self._session.get_status() - not in [TestrunStatuses.CANCELLED, - TestrunStatuses.COMPLIANT, - TestrunStatuses.NON_COMPLIANT + not in [TestrunStatus.CANCELLED, + TestrunStatus.COMPLIANT, + TestrunStatus.NON_COMPLIANT ]): response.status_code = 403 return self._generate_msg( @@ -544,9 +544,9 @@ async def edit_device(self, request: Request, response: Response): if (self._session.get_target_device() == device and self._session.get_status() - not in [TestrunStatuses.CANCELLED, - TestrunStatuses.COMPLIANT, - TestrunStatuses.NON_COMPLIANT + not in [TestrunStatus.CANCELLED, + TestrunStatus.COMPLIANT, + TestrunStatus.NON_COMPLIANT ]): response.status_code = 403 return self._generate_msg( diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 5ed0585e6..0d2a2dd4a 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -20,7 +20,7 @@ from fastapi.encoders import jsonable_encoder from common import util, logger, mqtt from common.risk_profile import RiskProfile -from common.config import TestrunStatuses, TestResults +from common.statuses import TestrunStatus, TestResult from net_orc.ip_control import IPControl # Certificate dependencies @@ -53,7 +53,7 @@ def wrapper(self, *args, **kwargs): result = method(self, *args, **kwargs) - if self.get_status() != TestrunStatuses.IDLE: + if self.get_status() != TestrunStatus.IDLE: self.get_mqtt_client().send_message( STATUS_TOPIC, jsonable_encoder(self.to_json()) @@ -81,7 +81,7 @@ class TestrunSession(): def __init__(self, root_dir): self._root_dir = root_dir - self._status = TestrunStatuses.IDLE + self._status = TestrunStatus.IDLE # Target test device self._device = None @@ -145,7 +145,7 @@ def __init__(self, root_dir): def start(self): self.reset() - self._status = TestrunStatuses.WAITING_FOR_DEVICE + self._status = TestrunStatus.WAITING_FOR_DEVICE self._started = datetime.datetime.now() def get_started(self): @@ -155,14 +155,14 @@ def get_finished(self): return self._finished def stop(self): - self.set_status(TestrunStatuses.STOPPING) + self.set_status(TestrunStatus.STOPPING) self.finish() def finish(self): # Set any in progress test results to Error for test_result in self._results: - if test_result.result == TestResults.IN_PROGRESS: - test_result.result = TestResults.ERROR + if test_result.result == TestResult.IN_PROGRESS: + test_result.result = TestResult.ERROR self._finished = datetime.datetime.now() @@ -369,7 +369,7 @@ def add_test_result(self, result): def set_test_result_error(self, result): """Set test result error""" - result.result = TestResults.ERROR + result.result = TestResult.ERROR self._results.append(result) def add_module_report(self, module_report): @@ -596,7 +596,7 @@ def delete_profile(self, profile): return False def reset(self): - self.set_status(TestrunStatuses.IDLE) + self.set_status(TestrunStatus.IDLE) self.set_target_device(None) self._report_url = None self._total_tests = 0 diff --git a/framework/python/src/common/config.py b/framework/python/src/common/statuses.py similarity index 96% rename from framework/python/src/common/config.py rename to framework/python/src/common/statuses.py index 4cae24cea..48bfcc1d6 100644 --- a/framework/python/src/common/config.py +++ b/framework/python/src/common/statuses.py @@ -14,7 +14,7 @@ """Testrun config""" -class TestrunStatuses: +class TestrunStatus: IDLE = "Idle" WAITING_FOR_DEVICE = "Waiting for Device" MONITORING = "Monitoring" @@ -25,7 +25,7 @@ class TestrunStatuses: STOPPING = "Stopping" -class TestResults: +class TestResult: IN_PROGRESS = "In Progress" COMPLIANT = "Compliant" NON_COMPLIANT = "Non-Compliant" diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py index 0c75cb90f..7fcac9f74 100644 --- a/framework/python/src/common/testreport.py +++ b/framework/python/src/common/testreport.py @@ -17,7 +17,7 @@ from weasyprint import HTML from io import BytesIO from common import util -from common.config import TestResults, TestrunStatuses +from common.statuses import TestResult, TestrunStatus import base64 import os from test_orc.test_case import TestCase @@ -44,7 +44,7 @@ class TestReport(): """Represents a previous Testrun report.""" def __init__(self, - status=TestrunStatuses.NON_COMPLIANT, + status=TestrunStatus.NON_COMPLIANT, started=None, finished=None, total_tests=0): @@ -431,7 +431,7 @@ def generate_results(self, json_data, page_num): successful_tests = 0 for test in json_data['tests']['results']: - if test['result'] != TestResults.ERROR: + if test['result'] != TestResult.ERROR: successful_tests += 1 result_list = f''' @@ -458,15 +458,15 @@ def generate_results(self, json_data, page_num): return result_list def generate_result(self, result): - if result['result'] == TestResults.NON_COMPLIANT: + if result['result'] == TestResult.NON_COMPLIANT: result_class = 'result-test-result-non-compliant' - elif result['result'] == TestResults.COMPLIANT: + elif result['result'] == TestResult.COMPLIANT: result_class = 'result-test-result-compliant' - elif result['result'] == TestResults.ERROR: + elif result['result'] == TestResult.ERROR: result_class = 'result-test-result-error' - elif result['result'] == TestResults.FEATURE_NOT_DETECTED: + elif result['result'] == TestResult.FEATURE_NOT_DETECTED: result_class = 'result-test-result-feature-not-detected' - elif result['result'] == TestResults.INFORMATIONAL: + elif result['result'] == TestResult.INFORMATIONAL: result_class = 'result-test-result-informational' else: result_class = 'result-test-result-skipped' @@ -616,7 +616,7 @@ def generate_device_module_label(self, module, enabled): return label def generate_result_summary(self, json_data): - if json_data['status'] == TestResults.COMPLIANT: + if json_data['status'] == TestResult.COMPLIANT: result_summary = '''
''' else: diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index 1c8b56704..ed30dd6a6 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -31,7 +31,7 @@ from common.device import Device from common.session import TestrunSession from common.testreport import TestReport -from common.config import TestrunStatuses +from common.statuses import TestrunStatus from api.api import Api from net_orc.listener import NetworkEvent from net_orc import network_orchestrator as net_orc @@ -373,7 +373,7 @@ def stop(self): self._stop_tests() self._stop_network(kill=True) - self.get_session().set_status(TestrunStatuses.CANCELLED) + self.get_session().set_status(TestrunStatus.CANCELLED) def _register_exits(self): signal.signal(signal.SIGINT, self._exit_handler) @@ -455,12 +455,12 @@ def _device_discovered(self, mac_addr): def _device_stable(self, mac_addr): # Do not continue testing if Testrun has cancelled during monitor phase - if self.get_session().get_status() == TestrunStatuses.CANCELLED: + if self.get_session().get_status() == TestrunStatus.CANCELLED: self._stop_network() return LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.') - self._set_status(TestrunStatuses.IN_PROGRESS) + self._set_status(TestrunStatus.IN_PROGRESS) result = self._test_orc.run_test_modules() if result is not None: diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 06c7788d7..8defb855c 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -25,7 +25,7 @@ import traceback from docker.types import Mount from common import logger, util, mqtt -from common.config import TestrunStatuses +from common.statuses import TestrunStatus from net_orc.listener import Listener from net_orc.network_event import NetworkEvent from net_orc.network_validator import NetworkValidator @@ -217,7 +217,7 @@ def _device_discovered(self, mac_addr): if device.ip_addr is None: LOGGER.info( f'Timed out whilst waiting for {mac_addr} to obtain an IP address') - self._session.set_status(TestrunStatuses.CANCELLED) + self._session.set_status(TestrunStatus.CANCELLED) return LOGGER.info( f'Device with mac addr {device.mac_addr} has obtained IP address ' @@ -277,7 +277,7 @@ def _dhcp_lease_ack(self, packet): def _start_device_monitor(self, device): """Start a timer until the steady state has been reached and callback the steady state method for this device.""" - self.get_session().set_status(TestrunStatuses.MONITORING) + self.get_session().set_status(TestrunStatus.MONITORING) self._monitor_packets = [] LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' f'for {str(self._session.get_monitor_period())} seconds') @@ -294,14 +294,14 @@ def _start_device_monitor(self, device): time.sleep(1) # Check Testrun hasn't been cancelled - if self._session.get_status() == TestrunStatuses.CANCELLED: + if self._session.get_status() == TestrunStatus.CANCELLED: sniffer.stop() return if not self._ip_ctrl.check_interface_status( self._session.get_device_interface()): sniffer.stop() - self._session.set_status(TestrunStatuses.CANCELLED) + self._session.set_status(TestrunStatus.CANCELLED) LOGGER.error('Device interface disconnected, cancelling Testrun') LOGGER.debug('Writing packets to monitor.pcap') @@ -811,9 +811,9 @@ def internet_conn_checker(self, mqtt_client: mqtt.MQTT, topic: str): # Only check if Testrun is running if self.get_session().get_status() not in [ - TestrunStatuses.WAITING_FOR_DEVICE, - TestrunStatuses.MONITORING, - TestrunStatuses.IN_PROGRESS + TestrunStatus.WAITING_FOR_DEVICE, + TestrunStatus.MONITORING, + TestrunStatus.IN_PROGRESS ]: message['connection'] = None diff --git a/framework/python/src/test_orc/test_case.py b/framework/python/src/test_orc/test_case.py index 750886d61..6c7aed6e8 100644 --- a/framework/python/src/test_orc/test_case.py +++ b/framework/python/src/test_orc/test_case.py @@ -14,7 +14,7 @@ """Represents an individual test case.""" from dataclasses import dataclass, field -from common.config import TestResults +from common.statuses import TestResult @dataclass @@ -25,7 +25,7 @@ class TestCase: # pylint: disable=too-few-public-methods,too-many-instance-attr description: str = "" expected_behavior: str = "" required_result: str = "Recommended" - result: str = TestResults.NON_COMPLIANT + result: str = TestResult.NON_COMPLIANT recommendations: list = field(default_factory=lambda: []) def to_dict(self): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index d6954179d..4e60e68da 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -23,7 +23,7 @@ from docker.types import Mount from common import logger, util from common.testreport import TestReport -from common.config import TestrunStatuses, TestResults +from common.statuses import TestrunStatus, TestResult from test_orc.module import TestModule from test_orc.test_case import TestCase import threading @@ -85,7 +85,7 @@ def run_test_modules(self): """Iterates through each test module and starts the container.""" # Do not start test modules if status is not in progress, e.g. Stopping - if self.get_session().get_status() != TestrunStatuses.IN_PROGRESS: + if self.get_session().get_status() != TestrunStatus.IN_PROGRESS: return device = self._session.get_target_device() @@ -110,7 +110,7 @@ def run_test_modules(self): test_copy = copy.deepcopy(test) # Set result to Not Started - test_copy.result = TestResults.NOT_STARTED + test_copy.result = TestResult.NOT_STARTED # We don't want steps to resolve for not started tests if hasattr(test_copy, "recommendations"): @@ -136,8 +136,8 @@ def run_test_modules(self): self._session.finish() # Do not carry on (generating a report) if Testrun has been stopped - if self.get_session().get_status() != TestrunStatuses.IN_PROGRESS: - return TestrunStatuses.CANCELLED + if self.get_session().get_status() != TestrunStatus.IN_PROGRESS: + return TestrunStatus.CANCELLED report = TestReport() report.from_json(self._generate_report()) @@ -204,16 +204,16 @@ def _generate_report(self): return report def _calculate_result(self): - result = TestResults.COMPLIANT + result = TestResult.COMPLIANT for test_result in self._session.get_test_results(): # Check Required tests if (test_result.required_result.lower() == "required" and test_result.result.lower() != "compliant"): - result = TestResults.NON_COMPLIANT + result = TestResult.NON_COMPLIANT # Check Required if Applicable tests elif (test_result.required_result.lower() == "required if applicable" and test_result.result.lower() == "non-compliant"): - result = TestResults.NON_COMPLIANT + result = TestResult.NON_COMPLIANT return result def _cleanup_old_test_results(self, device): @@ -380,7 +380,7 @@ def _run_test_module(self, module): """Start the test container and extract the results.""" # Check that Testrun is not stopping - if self.get_session().get_status() != TestrunStatuses.IN_PROGRESS: + if self.get_session().get_status() != TestrunStatus.IN_PROGRESS: return device = self._session.get_target_device() @@ -394,11 +394,11 @@ def _run_test_module(self, module): if not self._net_orc.is_device_connected(): LOGGER.error("Device was disconnected") self._set_test_modules_error(current_test) - self._session.set_status(TestrunStatuses.CANCELLED) + self._session.set_status(TestrunStatus.CANCELLED) return test_copy = copy.deepcopy(test) - test_copy.result = TestResults.IN_PROGRESS + test_copy.result = TestResult.IN_PROGRESS # We don't want steps to resolve for in progress tests if hasattr(test_copy, "recommendations"): @@ -494,7 +494,7 @@ def _run_test_module(self, module): log_thread.start() while (status == "running" - and self._session.get_status() == TestrunStatuses.IN_PROGRESS): + and self._session.get_status() == TestrunStatus.IN_PROGRESS): if time.time() > test_module_timeout: LOGGER.error("Module timeout exceeded, killing module: " + module.name) self._stop_module(module=module, kill=True) @@ -507,7 +507,7 @@ def _run_test_module(self, module): f.write(line + "\n") # Check that Testrun has not been stopped whilst this module was running - if self.get_session().get_status() == TestrunStatuses.STOPPING: + if self.get_session().get_status() == TestrunStatus.STOPPING: # Discard results for this module LOGGER.info(f"Test module {module.name} has forcefully quit") return @@ -535,11 +535,11 @@ def _run_test_module(self, module): result=test_result["result"]) # Any informational test should always report informational - if test_case.required_result == TestResults.INFORMATIONAL: - test_case.result = TestResults.INFORMATIONAL + if test_case.required_result == TestResult.INFORMATIONAL: + test_case.result = TestResult.INFORMATIONAL # Add steps to resolve if test is non-compliant - if (test_case.result == TestResults.NON_COMPLIANT and + if (test_case.result == TestResult.NON_COMPLIANT and "recommendations" in test_result): test_case.recommendations = test_result["recommendations"] else: diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 08a664a84..a3f9401e2 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -19,7 +19,7 @@ from datetime import datetime import traceback -from common.config import TestResults +from common.statuses import TestResult LOGGER = None RESULTS_DIR = '/runtime/output/' @@ -119,22 +119,22 @@ def run_tests(self): traceback.print_exc() else: LOGGER.error(f'Test {test["name"]} has not been implemented') - result = TestResults.ERROR, 'This test could not be found' + result = TestResult.ERROR, 'This test could not be found' else: LOGGER.debug(f'Test {test["name"]} is disabled') - result = TestResults.DISABLED, 'This test did not run because it is disabled' + result = TestResult.DISABLED, 'This test did not run because it is disabled' # Check if the test module has returned a result if result is not None: # Compliant or non-compliant as a boolean only if isinstance(result, bool): - test['result'] = TestResults.COMPLIANT if result else TestResults.NON_COMPLIANT + test['result'] = TestResult.COMPLIANT if result else TestResult.NON_COMPLIANT test['description'] = 'No description was provided for this test' else: # Error result if result[0] is None: - test['result'] = TestResults.ERROR + test['result'] = TestResult.ERROR if len(result) > 1: test['description'] = result[1] else: @@ -142,13 +142,13 @@ def run_tests(self): # Compliant / Non-Compliant result elif isinstance(result[0], bool): - test['result'] = TestResults.COMPLIANT if result[0] else TestResults.NON_COMPLIANT + test['result'] = TestResult.COMPLIANT if result[0] else TestResult.NON_COMPLIANT # Result may be a string, e.g Error, Feature Not Detected elif isinstance(result[0], str): test['result'] = result[0] else: LOGGER.error(f'Unknown result detected: {result[0]}') - test['result'] = TestResults.ERROR + test['result'] = TestResult.ERROR # Check that description is a string if isinstance(result[1], str): @@ -160,11 +160,11 @@ def run_tests(self): if len(result)>2: test['details'] = result[2] else: - test['result'] = TestResults.ERROR + test['result'] = TestResult.ERROR test['description'] = 'An error occured whilst running this test' # Remove the steps to resolve if compliant already - if (test['result'] == TestResults.COMPLIANT and 'recommendations' in test): + if (test['result'] == TestResult.COMPLIANT and 'recommendations' in test): test.pop('recommendations') test['end'] = datetime.now().isoformat() From ef9c0477b17b678188ccb8915f8aac58d33360f0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Nikitsin Date: Wed, 14 Aug 2024 18:18:27 +0200 Subject: [PATCH 9/9] pylint --- modules/test/base/python/src/test_module.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index a3f9401e2..2145c3e9d 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -122,14 +122,16 @@ def run_tests(self): result = TestResult.ERROR, 'This test could not be found' else: LOGGER.debug(f'Test {test["name"]} is disabled') - result = TestResult.DISABLED, 'This test did not run because it is disabled' + result = (TestResult.DISABLED, + 'This test did not run because it is disabled') # Check if the test module has returned a result if result is not None: # Compliant or non-compliant as a boolean only if isinstance(result, bool): - test['result'] = TestResult.COMPLIANT if result else TestResult.NON_COMPLIANT + test['result'] = (TestResult.COMPLIANT + if result else TestResult.NON_COMPLIANT) test['description'] = 'No description was provided for this test' else: # Error result @@ -142,7 +144,8 @@ def run_tests(self): # Compliant / Non-Compliant result elif isinstance(result[0], bool): - test['result'] = TestResult.COMPLIANT if result[0] else TestResult.NON_COMPLIANT + test['result'] = (TestResult.COMPLIANT + if result[0] else TestResult.NON_COMPLIANT) # Result may be a string, e.g Error, Feature Not Detected elif isinstance(result[0], str): test['result'] = result[0]