diff --git a/framework/python/src/common/testreport.py b/framework/python/src/common/testreport.py index 265b3f437..eab6194d6 100644 --- a/framework/python/src/common/testreport.py +++ b/framework/python/src/common/testreport.py @@ -463,6 +463,8 @@ def generate_result(self, result): result_class = 'result-test-result-compliant' elif result['result'] == 'Error': result_class = 'result-test-result-error' + elif result['result'] == 'Feature Not Detected': + result_class = 'result-test-result-feature-not-detected' else: result_class = 'result-test-result-skipped' @@ -1014,6 +1016,12 @@ def generate_css(self): left: 7.3in; } + .result-test-result-feature-not-detected { + background-color: #dadce0; + color: #8d5c00; + left: 6.92in; + } + .result-test-result-non-compliant { background-color: #FCE8E6; color: #C5221F; diff --git a/modules/devices/faux-dev/python/src/dns_check.py b/modules/devices/faux-dev/python/src/dns_check.py index be9c58d43..98dfd42e8 100644 --- a/modules/devices/faux-dev/python/src/dns_check.py +++ b/modules/devices/faux-dev/python/src/dns_check.py @@ -102,16 +102,16 @@ def _exec_tcpdump(self, tcpdump_filter): LOGGER.debug('tcpdump command: ' + command) - process = subprocess.Popen(command, + with subprocess.Popen(command, universal_newlines=True, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - text = str(process.stdout.read()).rstrip() + stderr=subprocess.PIPE) as process: + text = str(process.stdout.read()).rstrip() - LOGGER.debug('tcpdump response: ' + text) + LOGGER.debug('tcpdump response: ' + text) - if text: - return text.split('\n') + if text: + return text.split('\n') return [] diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index 471f22222..dc924cfeb 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -124,6 +124,7 @@ def run_tests(self): test['result'] = 'Compliant' if result else 'Non-Compliant' test['description'] = 'No description was provided for this test' else: + # TODO: This is assuming that result is an array but haven't checked # Skipped result if result[0] is None: test['result'] = 'Skipped' @@ -131,11 +132,12 @@ def run_tests(self): test['description'] = result[1] else: test['description'] = 'An error occured whilst running this test' + # Compliant / Non-Compliant result elif isinstance(result[0], bool): test['result'] = 'Compliant' if result[0] else 'Non-Compliant' - # Result may be a string, e.g error - elif result[0] == 'Error': + # Result may be a string, e.g Error, Feature Not Detected + elif isinstance(result[0], str): test['result'] = result[0] else: LOGGER.error(f'Unknown result detected: {result[0]}') diff --git a/modules/test/base/python/src/util.py b/modules/test/base/python/src/util.py index ba9c89d6d..006648037 100644 --- a/modules/test/base/python/src/util.py +++ b/modules/test/base/python/src/util.py @@ -28,20 +28,20 @@ def run_command(cmd, output=True): by any return code from the process other than zero.""" success = False - process = subprocess.Popen(shlex.split(cmd), + with subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if process.returncode != 0 and output: - err_msg = f'{stderr.strip()}. Code: {process.returncode}' - LOGGER.error('Command Failed: ' + cmd) - LOGGER.error('Error: ' + err_msg) - else: - success = True - LOGGER.debug('Command succeeded: ' + cmd) - if output: - out = stdout.strip().decode('utf-8') - LOGGER.debug('Command output: ' + out) - return out, stderr - else: - return success + stderr=subprocess.PIPE) as process: + stdout, stderr = process.communicate() + if process.returncode != 0 and output: + err_msg = f'{stderr.strip()}. Code: {process.returncode}' + LOGGER.error('Command Failed: ' + cmd) + LOGGER.error('Error: ' + err_msg) + else: + success = True + LOGGER.debug('Command succeeded: ' + cmd) + if output: + out = stdout.strip().decode('utf-8') + LOGGER.debug('Command output: ' + out) + return out, stderr + else: + return success diff --git a/modules/test/dns/conf/module_config.json b/modules/test/dns/conf/module_config.json index 4a0dc4930..13c9b3236 100644 --- a/modules/test/dns/conf/module_config.json +++ b/modules/test/dns/conf/module_config.json @@ -27,7 +27,7 @@ "name": "dns.network.from_dhcp", "test_description": "Verify the device allows for a DNS server to be entered automatically", "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server", - "required_result": "Roadmap", + "required_result": "Informational", "recommendations": [ "Install a DNS client that supports fetching DNS servers from DHCP options" ] diff --git a/modules/test/dns/python/src/dns_module.py b/modules/test/dns/python/src/dns_module.py index e9550663d..4d1c315a3 100644 --- a/modules/test/dns/python/src/dns_module.py +++ b/modules/test/dns/python/src/dns_module.py @@ -196,7 +196,6 @@ def _has_dns_traffic(self, tcpdump_filter): def _dns_network_from_dhcp(self): LOGGER.info('Running dns.network.from_dhcp') - result = None LOGGER.info('Checking DNS traffic for configured DHCP DNS server: ' + self._dns_server) @@ -213,25 +212,24 @@ def _dns_network_from_dhcp(self): if dns_packets_local or dns_packets_not_local: if dns_packets_not_local: - result = False, 'DNS traffic detected to non-DHCP provided server' + description = 'DNS traffic detected to non-DHCP provided server' else: LOGGER.info('DNS traffic detected only to configured DHCP DNS server') - result = True, 'DNS traffic detected only to DHCP provided server' + description = 'DNS traffic detected only to DHCP provided server' else: LOGGER.info('No DNS traffic detected from the device') - result = None, 'No DNS traffic detected from the device' - return result + description = 'No DNS traffic detected from the device' + return 'Informational', description def _dns_network_hostname_resolution(self): LOGGER.info('Running dns.network.hostname_resolution') - result = None LOGGER.info('Checking DNS traffic from device: ' + self._device_mac) # Check if the device DNS traffic tcpdump_filter = f'dst port 53 and ether src {self._device_mac}' - dns_packetes = self._has_dns_traffic(tcpdump_filter=tcpdump_filter) + dns_packets = self._has_dns_traffic(tcpdump_filter=tcpdump_filter) - if dns_packetes: + if dns_packets: LOGGER.info('DNS traffic detected from device') result = True, 'DNS traffic detected from device' else: @@ -239,20 +237,18 @@ def _dns_network_hostname_resolution(self): result = False, 'No DNS traffic detected from the device' return result - ## TODO: This test should always return 'Informational' result def _dns_mdns(self): LOGGER.info('Running dns.mdns') - result = None # Check if the device sends any MDNS traffic tcpdump_filter = f'udp port 5353 and ether src {self._device_mac}' - dns_packetes = self._has_dns_traffic(tcpdump_filter=tcpdump_filter) + dns_packets = self._has_dns_traffic(tcpdump_filter=tcpdump_filter) - if dns_packetes: + if dns_packets: LOGGER.info('MDNS traffic detected from device') - result = True, 'MDNS traffic detected from device' + result = 'Informational', 'MDNS traffic detected from device' else: LOGGER.info('No MDNS traffic detected from the device') - result = True, 'No MDNS traffic detected from the device' + result = 'Informational', 'No MDNS traffic detected from the device' return result def _exec_tcpdump(self, tcpdump_filter, capture_file): @@ -267,16 +263,16 @@ def _exec_tcpdump(self, tcpdump_filter, capture_file): LOGGER.debug('tcpdump command: ' + command) - process = subprocess.Popen(command, + with subprocess.Popen(command, universal_newlines=True, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - text = str(process.stdout.read()).rstrip() + stderr=subprocess.PIPE) as process: + text = str(process.stdout.read()).rstrip() - LOGGER.debug('tcpdump response: ' + text) + LOGGER.debug('tcpdump response: ' + text) - if text: - return text.split('\n') + if text: + return text.split('\n') return [] diff --git a/modules/test/ntp/python/src/ntp_module.py b/modules/test/ntp/python/src/ntp_module.py index 23c5cfd13..453c992e6 100644 --- a/modules/test/ntp/python/src/ntp_module.py +++ b/modules/test/ntp/python/src/ntp_module.py @@ -208,7 +208,6 @@ def extract_ntp_data(self): def _ntp_network_ntp_support(self): LOGGER.info('Running ntp.network.ntp_support') - result = None packet_capture = (rdpcap(STARTUP_CAPTURE_FILE) + rdpcap(MONITOR_CAPTURE_FILE) + rdpcap(NTP_SERVER_CAPTURE_FILE)) @@ -245,7 +244,6 @@ def _ntp_network_ntp_support(self): def _ntp_network_ntp_dhcp(self): LOGGER.info('Running ntp.network.ntp_dhcp') - result = None packet_capture = (rdpcap(STARTUP_CAPTURE_FILE) + rdpcap(MONITOR_CAPTURE_FILE) + rdpcap(NTP_SERVER_CAPTURE_FILE)) @@ -273,11 +271,12 @@ def _ntp_network_ntp_dhcp(self): result = False, ('Device sent NTP request to DHCP provided ' + 'server and non-DHCP provided server') elif ntp_to_remote: - result = False, 'Device sent NTP request to non-DHCP provided server' + result = ('Feature Not Detected', + 'Device sent NTP request to non-DHCP provided server') elif ntp_to_local: result = True, 'Device sent NTP request to DHCP provided server' else: - result = False, 'Device has not sent any NTP requests' + result = 'Feature Not Detected', 'Device has not sent any NTP requests' LOGGER.info(result[1]) return result diff --git a/modules/test/protocol/python/src/protocol_bacnet.py b/modules/test/protocol/python/src/protocol_bacnet.py index 8560ba00f..a987a9f4c 100644 --- a/modules/test/protocol/python/src/protocol_bacnet.py +++ b/modules/test/protocol/python/src/protocol_bacnet.py @@ -15,7 +15,10 @@ import BAC0 import logging -from BAC0.core.io.IOExceptions import UnknownPropertyError, ReadPropertyException, NoResponseFromController, DeviceNotConnected +from BAC0.core.io.IOExceptions import (UnknownPropertyError, + ReadPropertyException, + NoResponseFromController, + DeviceNotConnected) LOGGER = None BAC0_LOG = '/root/.BAC0/BAC0.log' @@ -54,23 +57,21 @@ def discover(self, local_ip=None): # Check if the device being tested is in the discovered devices list def validate_device(self, local_ip, device_ip): - result = None LOGGER.info('Validating BACnet device: ' + device_ip) self.discover(local_ip + '/24') - LOGGER.info('BACnet Devices Found: ' + str(len(self.devices))) + LOGGER.info('BACnet devices found: ' + str(len(self.devices))) if len(self.devices) > 0: - # Load a fail result initially and pass only - # if we can validate it's the right device responding - result = False, ('Could not confirm discovered BACnet device is the ' + - 'same as device being tested') + result = (False, + 'BACnet device was found but was not device under test') for device in self.devices: address = device[2] LOGGER.info('Checking device: ' + str(device)) if device_ip in address: - result = True, 'Device IP matches discovered device' + result = True, 'BACnet device discovered' break else: - result = None, 'BACnet discovery could not resolve any devices' + result = ('Feature Not Detected', + 'BACnet device could not be discovered') if result is not None: LOGGER.info(result[1]) return result @@ -85,7 +86,7 @@ def validate_protocol_version(self, device_ip, device_id): protocol_version = f'{version}.{revision}' result = True result_description = ( - f'BACnet protocol version detected: {protocol_version}') + f'Device uses BACnet version {protocol_version}') except (UnknownPropertyError, ReadPropertyException, NoResponseFromController, DeviceNotConnected) as e: result = False diff --git a/modules/test/protocol/python/src/protocol_modbus.py b/modules/test/protocol/python/src/protocol_modbus.py index d9a74c834..925e9517a 100644 --- a/modules/test/protocol/python/src/protocol_modbus.py +++ b/modules/test/protocol/python/src/protocol_modbus.py @@ -267,7 +267,7 @@ def validate_device(self): compliant = (holding_reg is not None or input_reg is not None or coils is not None or discrete_inputs is not None) else: - compliant = False + compliant = None details = 'Failed to establish Modbus connection to device' result = compliant, details return result diff --git a/modules/test/protocol/python/src/protocol_module.py b/modules/test/protocol/python/src/protocol_module.py index 932bc7702..bfb248cd5 100644 --- a/modules/test/protocol/python/src/protocol_module.py +++ b/modules/test/protocol/python/src/protocol_module.py @@ -25,6 +25,7 @@ class ProtocolModule(TestModule): """Protocol Test module""" def __init__(self, module): + self._supports_bacnet = False super().__init__(module_name=module, log_name=LOG_NAME) global LOGGER LOGGER = self._get_logger() @@ -48,8 +49,10 @@ def _protocol_valid_bacnet(self): if local_address: result = self._bacnet.validate_device(local_address, self._device_ipv4_addr) + if result[0]: + self._supports_bacnet = True else: - result = None, 'Could not resolve test container IP for BACnet discovery' + result = 'Error', 'Failed to perform BACnet discovery' return result def _protocol_bacnet_version(self): @@ -59,20 +62,23 @@ def _protocol_bacnet_version(self): this test can pass. """ LOGGER.info('Running protocol.bacnet.version') - result_status = None - result_description = '' + result_status = 'Feature Not Detected' + result_description = 'Device did not respond to BACnet discovery' + + # Do not run test if device does not support BACnet + if not self._supports_bacnet: + return result_status, result_description if len(self._bacnet.devices) > 0: for device in self._bacnet.devices: - LOGGER.info(f'Checking BACnet version for device: {device}') if self._device_ipv4_addr in device[2]: + LOGGER.debug(f'Checking BACnet version for device: {device}') result_status, result_description = \ - self._bacnet.validate_protocol_version( device[2], device[3]) + self._bacnet.validate_protocol_version(device[2], device[3]) break else: - LOGGER.info('Device does not match expected IP address, skipping') - else: - result_description = 'No BACnet devices discovered.' + LOGGER.debug('Device does not match expected IP address, skipping') + LOGGER.info(result_description) return result_status, result_description @@ -81,15 +87,16 @@ def _protocol_valid_modbus(self, config): # Extract basic device connection information modbus = Modbus(log=LOGGER, device_ip=self._device_ipv4_addr, config=config) results = modbus.validate_device() + # Determine results and return proper messaging and details - description = '' if results[0] is None: - description = 'No modbus connection could be made' + result = ('Feature Not Detected', + 'Device did not respond to Modbus connection') elif results[0]: - description = 'Valid modbus communication detected' + result = True, 'Valid modbus communication detected' else: - description = 'Failed to confirm valid modbus communication' - return results[0], description, results[1] + result = False, 'Failed to confirm valid modbus communication' + return result, results[1] def get_local_ip(self, interface_name): try: diff --git a/modules/test/tls/python/src/run.py b/modules/test/tls/python/src/run.py index 89de9f65e..78296f0c1 100644 --- a/modules/test/tls/python/src/run.py +++ b/modules/test/tls/python/src/run.py @@ -66,4 +66,4 @@ def run(): if __name__ == '__main__': - run() + run() \ No newline at end of file diff --git a/modules/test/tls/python/src/tls_module.py b/modules/test/tls/python/src/tls_module.py index 0994d5b22..078081f32 100644 --- a/modules/test/tls/python/src/tls_module.py +++ b/modules/test/tls/python/src/tls_module.py @@ -50,7 +50,6 @@ def __init__(self, self._tls_util = TLSUtil(LOGGER) # def generate_module_report(self): - # html_content = '