From 1ebc63403496f6f4ecee26f9045bb8510dcb5990 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Wed, 16 Aug 2023 16:16:53 +0100 Subject: [PATCH] Add required result to module configs --- framework/python/src/common/device.py | 2 +- framework/python/src/common/session.py | 2 +- framework/python/src/test_orc/module.py | 11 ++-- framework/python/src/test_orc/test_case.py | 26 +++++++++ .../python/src/test_orc/test_orchestrator.py | 54 +++++++++++++++++-- modules/test/baseline/conf/module_config.json | 6 +-- modules/test/conn/conf/module_config.json | 27 +++++----- modules/test/dns/conf/module_config.json | 9 ++-- modules/test/nmap/conf/module_config.json | 52 +++++++++--------- modules/test/ntp/conf/module_config.json | 4 +- modules/test/tls/conf/module_config.json | 8 +-- 11 files changed, 139 insertions(+), 62 deletions(-) create mode 100644 framework/python/src/test_orc/test_case.py diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py index e2552d75a..c79cee84f 100644 --- a/framework/python/src/common/device.py +++ b/framework/python/src/common/device.py @@ -29,7 +29,7 @@ class Device(): device_folder: str = None max_device_reports: int = None - def to_json(self): + def to_dict(self): device_json = {} device_json['mac_addr'] = self.mac_addr device_json['manufacturer'] = self.manufacturer diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 13e4b09fb..18590472c 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -131,7 +131,7 @@ def get_monitor_period(self): def get_startup_timeout(self): return self._config.get(STARTUP_TIMEOUT_KEY) - + def get_max_device_reports(self): return self._config.get(MAX_DEVICE_REPORTS_KEY) diff --git a/framework/python/src/test_orc/module.py b/framework/python/src/test_orc/module.py index 185940dd8..ba22e9911 100644 --- a/framework/python/src/test_orc/module.py +++ b/framework/python/src/test_orc/module.py @@ -12,31 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Represemts a test module.""" -from dataclasses import dataclass +"""Represents a test module.""" +from dataclasses import dataclass, field from docker.models.containers import Container - @dataclass class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Represents a test module.""" + # General test module information name: str = None display_name: str = None description: str = None + tests: list = field(default_factory=lambda: []) + # Docker settings build_file: str = None container: Container = None container_name: str = None image_name: str = None enable_container: bool = True network: bool = True - timeout: int = 60 # Absolute path dir: str = None dir_name: str = None - #Set IP Index for all test modules + # Set IP Index for all test modules ip_index: str = 9 diff --git a/framework/python/src/test_orc/test_case.py b/framework/python/src/test_orc/test_case.py new file mode 100644 index 000000000..7c9eb6c20 --- /dev/null +++ b/framework/python/src/test_orc/test_case.py @@ -0,0 +1,26 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Represents an individual test case.""" +from dataclasses import dataclass + + +@dataclass +class TestCase: # pylint: disable=too-few-public-methods,too-many-instance-attributes + """Represents a test case.""" + + name: str = "test.undefined" + description: str = "" + expected_behavior: str = "" + required_result: str = "Recommended" diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index b9353c995..6c5bfba1c 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -22,6 +22,7 @@ from docker.types import Mount from common import logger, util from test_orc.module import TestModule +from test_orc.test_case import TestCase LOG_NAME = "test_orc" LOGGER = logger.get_logger("test_orc") @@ -95,12 +96,12 @@ def _generate_report(self): # We need to know the required result of each test report = {} - report["device"] = self._session.get_target_device().to_json() + report["device"] = self._session.get_target_device().to_dict() report["started"] = self._session.get_started().strftime( "%Y-%m-%d %H:%M:%S") report["finished"] = self._session.get_finished().strftime( "%Y-%m-%d %H:%M:%S") - report["status"] = self._session.get_status() + report["status"] = self._calculate_result() report["results"] = self._session.get_test_results() out_file = os.path.join( self._root_path, RUNTIME_DIR, @@ -112,6 +113,15 @@ def _generate_report(self): util.run_command(f"chown -R {self._host_user} {out_file}") return report + def _calculate_result(self): + result = "Compliant" + for test_result in self._session.get_test_results(): + test_case = self.get_test_case(test_result["name"]) + if (test_case.required_result.lower() == "required" + and test_result["result"].lower() == "non-compliant"): + result = "non-compliant" + return result + def _cleanup_old_test_results(self, device): if device.max_device_reports is not None: @@ -336,7 +346,7 @@ def _load_test_modules(self): def _load_test_module(self, module_dir): """Import module configuration from module_config.json.""" - LOGGER.debug("Loading test module " + module_dir) + LOGGER.debug(f"Loading test module {module_dir}") modules_dir = os.path.join(self._path, TEST_MODULES_DIR) @@ -355,6 +365,21 @@ def _load_test_module(self, module_dir): module.container_name = "tr-ct-" + module.dir_name + "-test" module.image_name = "test-run/" + module.dir_name + "-test" + # Load test cases + if "tests" in module_json["config"]: + for test_case_json in module_json["config"]["tests"]: + try: + test_case = TestCase( + name=test_case_json["name"], + description=test_case_json["description"], + expected_behavior=test_case_json["expected_behavior"], + required_result=test_case_json["required_result"] + ) + module.tests.append(test_case) + except Exception as error: + LOGGER.debug("Failed to load test case. See error for details") + LOGGER.error(error) + if "timeout" in module_json["config"]["docker"]: module.timeout = module_json["config"]["docker"]["timeout"] @@ -367,6 +392,7 @@ def _load_test_module(self, module_dir): if "network" in module_json["config"]: module.network = module_json["config"]["network"] + # Ensure container is built after any dependencies if "depends_on" in module_json["config"]["docker"]: depends_on_module = module_json["config"]["docker"]["depends_on"] if self._get_test_module(depends_on_module) is None: @@ -417,3 +443,25 @@ def _stop_module(self, module, kill=False): LOGGER.debug("Container stopped:" + module.container_name) except docker.errors.NotFound: pass + + def get_test_modules(self): + return self._test_modules + + def get_test_module(self, name): + for test_module in self.get_test_modules(): + if test_module.name == name: + return test_module + return None + + def get_test_cases(self): + test_cases = [] + for test_module in self.get_test_modules(): + for test_case in test_module.tests: + test_cases.append(test_case) + return test_cases + + def get_test_case(self, name): + for test_case in self.get_test_cases(): + if test_case.name == name: + return test_case + return None diff --git a/modules/test/baseline/conf/module_config.json b/modules/test/baseline/conf/module_config.json index f4daf0e36..83b920ea6 100644 --- a/modules/test/baseline/conf/module_config.json +++ b/modules/test/baseline/conf/module_config.json @@ -16,19 +16,19 @@ "name": "baseline.pass", "description": "Simulate a compliant test", "expected_behavior": "A compliant test result is generated", - "short_description": "A compliant test result is generated" + "required_result": "Required" }, { "name": "baseline.fail", "description": "Simulate a non-compliant test", "expected_behavior": "A non-compliant test result is generated", - "short_description": "A non-compliant test result is generated" + "required_result": "Recommended" }, { "name": "baseline.skip", "description": "Simulate a skipped test", "expected_behavior": "A skipped test result is generated", - "short_description": "A skipped test result is generated" + "required_result": "Roadmap" } ] } diff --git a/modules/test/conn/conf/module_config.json b/modules/test/conn/conf/module_config.json index d721b1616..c358ba1c2 100644 --- a/modules/test/conn/conf/module_config.json +++ b/modules/test/conn/conf/module_config.json @@ -17,37 +17,37 @@ "name": "connection.dhcp.disconnect", "description": "The device under test has received an IP address from the DHCP server and responds to an ICMP echo (ping) request", "expected_behavior": "The device is not setup with a static IP address. The device accepts an IP address from a DHCP server (RFC 2131) and responds succesfully to an ICMP echo (ping) request.", - "short_description": "Device has received an IP address after port disconnect" + "required_result": "Required" }, { "name": "connection.dhcp.disconnect_ip_change", "description": "Update device IP on the DHCP server and reconnect the device. Does the device receive the new IP address?", "expected_behavior": "Device recieves a new IP address within the range that is specified on the DHCP server. Device should respond to aping on this new address.", - "short_description": "Device has received new IP address after port disconnect" + "required_result": "Required" }, { "name": "connection.dhcp_address", "description": "The device under test has received an IP address from the DHCP server and responds to an ICMP echo (ping) request", "expected_behavior": "The device is not setup with a static IP address. The device accepts an IP address from a DHCP server (RFC 2131) and responds succesfully to an ICMP echo (ping) request.", - "short_description": "Device has received a DHCP provided IP address" + "required_result": "Required" }, { "name": "connection.mac_address", "description": "Check and note device physical address.", "expected_behavior": "N/A", - "short_description": "Device MAC address resolved" + "required_result": "Required" }, { "name": "connection.mac_oui", "description": "The device under test hs a MAC address prefix that is registered against a known manufacturer.", "expected_behavior": "The MAC address prefix is registered in the IEEE Organizationally Unique Identifier database.", - "short_description": "OUI for MAC address resolved" + "required_result": "Required" }, { "name": "connection.private_address", "description": "The device under test accepts an IP address that is compliant with RFC 1918 Address Allocation for Private Internets.", "expected_behavior": "The device under test accepts IP addresses within all ranges specified in RFC 1918 and communicates using these addresses. The Internet Assigned Numbers Authority (IANA) has reserved the following three blocks of the IP address space for private internets. 10.0.0.0 - 10.255.255.255.255 (10/8 prefix). 172.16.0.0 - 172.31.255.255 (172.16/12 prefix). 192.168.0.0 - 192.168.255.255 (192.168/16 prefix)", - "short_description": "Device supports private addresses", + "required_result": "Required", "config": { "ranges": [ { @@ -69,7 +69,7 @@ "name": "connection.shared_address", "description": "Ensure the device supports RFC 6598 IANA-Reserved IPv4 Prefix for Shared Address Space", "expected_behavior": "The device under test accepts IP addresses within the ranges specified in RFC 6598 and communicates using these addresses", - "short_description": "Device supports shared address space", + "required_result": "Required", "config": { "ranges": [ { @@ -83,6 +83,7 @@ "name": "connection.private_address", "description": "The device under test accepts an IP address that is compliant with RFC 1918 Address Allocation for Private Internets.", "expected_behavior": "The device under test accepts IP addresses within all ranges specified in RFC 1918 and communicates using these addresses. The Internet Assigned Numbers Authority (IANA) has reserved the following three blocks of the IP address space for private internets. 10.0.0.0 - 10.255.255.255.255 (10/8 prefix). 172.16.0.0 - 172.31.255.255 (172.16/12 prefix). 192.168.0.0 - 192.168.255.255 (192.168/16 prefix)", + "required_result": "Required", "config": [ { "start": "10.0.0.100", @@ -102,37 +103,37 @@ "name": "connection.single_ip", "description": "The network switch port connected to the device reports only one IP address for the device under test.", "expected_behavior": "The device under test does not behave as a network switch and only requets one IP address. This test is to avoid that devices implement network switches that allow connecting strings of daisy chained devices to one single network port, as this would not make 802.1x port based authentication possible.", - "short_description": "Device only reports one IP address" + "required_result": "Required" }, { "name": "connection.target_ping", "description": "The device under test responds to an ICMP echo (ping) request.", "expected_behavior": "The device under test responds to an ICMP echo (ping) request.", - "short_description": "Device responds to a ping request" + "required_result": "Required" }, { "name": "connection.ipaddr.ip_change", "description": "The device responds to a ping (ICMP echo request) to the new IP address it has received after the initial DHCP lease has expired.", "expected_behavior": "If the lease expires before the client receiveds a DHCPACK, the client moves to INIT state, MUST immediately stop any other network processing and requires network initialization parameters as if the client were uninitialized. If the client then receives a DHCPACK allocating the client its previous network addres, the client SHOULD continue network processing. If the client is given a new network address, it MUST NOT continue using the previous network address and SHOULD notify the local users of the problem.", - "short_description": "Device receives an IP change from the DHCP server" + "required_result": "Required" }, { "name": "connection.ipaddr.dhcp_failover", "description": "The device has requested a DHCPREQUEST/REBIND to the DHCP failover server after the primary DHCP server has been brought down.", "expected_behavior": "", - "short_description": "Device receives IP address from primary and failover DHCP servers" + "required_result": "Required" }, { "name": "connection.ipv6_slaac", "description": "The device forms a valid IPv6 address as a combination of the IPv6 router prefix and the device interface identifier", "expected_behavior": "The device under test complies with RFC4862 and forms a valid IPv6 SLAAC address", - "short_description": "Device uses an IPv6 address using SLAAC" + "required_result": "Required" }, { "name": "connection.ipv6_ping", "description": "The device responds to an IPv6 ping (ICMPv6 Echo) request to the SLAAC address", "expected_behavior": "The device responds to the ping as per RFC4443", - "short_description": "Device responds to an IPv6 SLAAC address ping request" + "required_result": "Required" } ] } diff --git a/modules/test/dns/conf/module_config.json b/modules/test/dns/conf/module_config.json index b5e3c8420..e00061047 100644 --- a/modules/test/dns/conf/module_config.json +++ b/modules/test/dns/conf/module_config.json @@ -13,21 +13,22 @@ }, "tests":[ { - "name": "dns.network.from_device", + "name": "dns.network.hostname_resolution", "description": "Verify the device sends DNS requests", "expected_behavior": "The device sends DNS requests.", - "short_description": "The device sends DNS requests." + "required_result": "Required" }, { "name": "dns.network.from_dhcp", "description": "Verify the device allows for a DNS server to be entered automatically", "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server", - "short_description": "The device sends DNS requests to local DNS server." + "required_result": "Roadmap" }, { "name": "dns.mdns", "description": "If the device has MDNS (or any kind of IP multicast), can it be disabled", - "short_description": "MDNS traffic detected from device" + "expected_behavior": "Device may send MDNS requests", + "required_result": "Recommended" } ] } diff --git a/modules/test/nmap/conf/module_config.json b/modules/test/nmap/conf/module_config.json index b03e9511c..8a90febc1 100644 --- a/modules/test/nmap/conf/module_config.json +++ b/modules/test/nmap/conf/module_config.json @@ -16,7 +16,6 @@ "name": "security.nmap.ports", "description": "Run an nmap scan of open ports", "expected_behavior": "Report all open ports", - "short_description": "NMAP scan reports no unallowed ports open", "config": { "security.services.ftp": { "tcp_ports": { @@ -30,9 +29,10 @@ } }, "description": "Check FTP port 20/21 is disabled and FTP is not running on any port", - "expected_behavior": "There is no FTP service running on any port" + "expected_behavior": "There is no FTP service running on any port", + "required_result": "Required" }, - "security.services.ssh": { + "security.ssh.version": { "tcp_ports": { "22": { "allowed": true, @@ -40,8 +40,9 @@ "version": "2.0" } }, - "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", - "expected_behavior": "There is no FTP service running on any port" + "description": "If the device is running a SSH server ensure it is SSHv2", + "expected_behavior": "SSH server is not running or server is SSHv2", + "required_result": "Required" }, "security.services.telnet": { "tcp_ports": { @@ -51,7 +52,8 @@ } }, "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", - "expected_behavior": "There is no FTP service running on any port" + "expected_behavior": "There is no FTP service running on any port", + "required_result": "Required" }, "security.services.smtp": { "tcp_ports": { @@ -68,8 +70,9 @@ "description": "Simple Mail Transfer Protocol via TLS (SMTPS) Server" } }, - "description": "Check SMTP port 25 is disabled and ports 465 or 587 with SSL encryption are (not?) enabled and SMTP is not running on any port.", - "expected_behavior": "There is no smtp service running on any port" + "description": "Check SMTP ports 25, 465 and 587 are not enabled and SMTP is not running on any port.", + "expected_behavior": "There is no smtp service running on any port", + "required_result": "Required" }, "security.services.http": { "tcp_ports": { @@ -82,7 +85,8 @@ } }, "description": "Check that there is no HTTP server running on any port", - "expected_behavior": "Device is unreachable on port 80 (or any other port) and only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)" + "expected_behavior": "Device is unreachable on port 80 (or any other port) and only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)", + "required_result": "Required" }, "security.services.pop": { "tcp_ports": { @@ -92,7 +96,8 @@ } }, "description": "Check POP port 110 is disalbed and POP is not running on any port", - "expected_behavior": "There is no pop service running on any port" + "expected_behavior": "There is no pop service running on any port", + "required_result": "Required" }, "security.services.imap": { "tcp_ports": { @@ -102,7 +107,8 @@ } }, "description": "Check IMAP port 143 is disabled and IMAP is not running on any port", - "expected_behavior": "There is no imap service running on any port" + "expected_behavior": "There is no imap service running on any port", + "required_result": "Required" }, "security.services.snmpv3": { "tcp_ports": { @@ -126,17 +132,8 @@ } }, "description": "Check SNMP port 161/162 is disabled. If SNMP is an essential service, check it supports version 3", - "expected_behavior": "Device is unreachable on port 161 (or any other port) and device is unreachable on port 162 (or any other port) unless SNMP is essential in which case it is SNMPv3 is used." - }, - "security.services.https": { - "tcp_ports": { - "80": { - "allowed": false, - "description": "Administrative Secure Web-Server" - } - }, - "description": "Check that if there is a web server running it is running on a secure port.", - "expected_behavior": "Device only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)" + "expected_behavior": "Device is unreachable on port 161 (or any other port) and device is unreachable on port 162 (or any other port) unless SNMP is essential in which case it is SNMPv3 is used.", + "required_result": "Required" }, "security.services.vnc": { "tcp_ports": { @@ -150,7 +147,8 @@ } }, "description": "Check VNC is disabled on any port", - "expected_behavior": "Device cannot be accessed /connected to via VNc on any port" + "expected_behavior": "Device cannot be accessed /connected to via VNC on any port", + "required_result": "Required" }, "security.services.tftp": { "udp_ports": { @@ -160,9 +158,10 @@ } }, "description": "Check TFTP port 69 is disabled (UDP)", - "expected_behavior": "There is no tftp service running on any port" + "expected_behavior": "There is no tftp service running on any port", + "required_result": "Required" }, - "security.services.ntp": { + "ntp.network.ntp_server": { "udp_ports": { "123": { "allowed": false, @@ -172,7 +171,8 @@ "description": "Check NTP port 123 is disabled and the device is not operating as an NTP server", "expected_behavior": "The device dos not respond to NTP requests when it's IP is set as the NTP server on another device" } - } + }, + "required_result": "Required" } ] } diff --git a/modules/test/ntp/conf/module_config.json b/modules/test/ntp/conf/module_config.json index c20d2067b..a1a297f06 100644 --- a/modules/test/ntp/conf/module_config.json +++ b/modules/test/ntp/conf/module_config.json @@ -16,13 +16,13 @@ "name": "ntp.network.ntp_support", "description": "Does the device request network time sync as client as per RFC 5905 - Network Time Protocol Version 4: Protocol and Algorithms Specification", "expected_behavior": "The device sends an NTPv4 request to the configured NTP server.", - "short_description": "The device sends NTPv4 requests" + "required_result": "Required" }, { "name": "ntp.network.ntp_dhcp", "description": "Accept NTP address over DHCP", "expected_behavior": "Device can accept NTP server address, provided by the DHCP server (DHCP OFFER PACKET)", - "short_descriiption": "Accepts NTP address over DHCP" + "required_result": "Roadmap" } ] } diff --git a/modules/test/tls/conf/module_config.json b/modules/test/tls/conf/module_config.json index f71f39914..7f0305d19 100644 --- a/modules/test/tls/conf/module_config.json +++ b/modules/test/tls/conf/module_config.json @@ -16,25 +16,25 @@ "name": "security.tls.v1_2_server", "description": "Check the device web server TLS 1.2 & certificate is valid", "expected_behavior": "TLS 1.2 certificate is issued to the web browser client when accessed", - "short_description": "TLS 1.2 server certificate is valid" + "required_result": "Required" }, { "name": "security.tls.v1_3_server", "description": "Check the device web server TLS 1.3 & certificate is valid", "expected_behavior": "TLS 1.3 certificate is issued to the web browser client when accessed", - "short_description": "TLS 1.3 server certificate is valid" + "required_result": "Recommended" }, { "name": "security.tls.v1_2_client", "description": "Device uses TLS with connection to an external service on port 443 (or any other port which could be running the webserver-HTTPS)", "expected_behavior": "The packet indicates a TLS connection with at least TLS 1.2 and support for ECDH and ECDSA ciphers", - "short_description": "TLS 1.2 outbound connection valid" + "required_result": "Required" }, { "name": "security.tls.v1_3_client", "description": "Device uses TLS with connection to an external service on port 443 (or any other port which could be running the webserver-HTTPS)", "expected_behavior": "The packet indicates a TLS connection with at least TLS 1.3", - "short_description": "TLS 1.3 outbound connection valid" + "required_result": "Recommended" } ] }