Initial CI testing for tests

.github/workflows/testing.yml

@@ -7,7 +7,7 @@ on:
     - cron: '0 13 * * *'
 
 jobs:
-  testrun:
+  testrun_baseline:
     name:  Baseline
     runs-on: ubuntu-20.04
     timeout-minutes: 20
@@ -17,11 +17,21 @@ jobs:
       - name: Run tests
         shell: bash {0}
         run: testing/test_baseline
-
+
+  testrun_tests:
+    name:  Tests
+    runs-on: ubuntu-20.04
+    timeout-minutes: 40
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.4
+      - name: Run tests
+        shell: bash {0}
+        run: testing/test_tests
   pylint:
     name:  Pylint
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
     steps:
       - name: Checkout source
         uses: actions/checkout@v2.3.4

testing/docker/ci_test_device1/Dockerfile

@@ -0,0 +1,11 @@
+
+FROM ubuntu:jammy
+
+#Update and get all additional requirements not contained in the base image 
+RUN apt-get update && apt-get -y upgrade
+
+RUN apt-get update && apt-get install -y isc-dhcp-client ntpdate coreutils moreutils inetutils-ping curl jq dnsutils openssl netcat-openbsd 
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

testing/docker/ci_test_device1/entrypoint.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+ip a
+
+declare -A options
+for option in $*; do
+    if [[ $option == *"="* ]]; then
+        k=$(echo $option | cut -d'=' -f1)
+        v=$(echo $option | cut -d'=' -f2)
+        options[$k]=$v
+    else
+        options[$option]=$option
+    fi
+done
+
+OUT=/out/testrun_ci.json
+
+NTP_SERVER=10.10.10.5
+DNS_SERVER=10.10.10.4
+
+function wout(){
+    temp=${1//./\".\"}
+    key=${temp:1}\"
+    echo $key
+    value=$2
+    jq "$key+=\"$value\"" $OUT | sponge $OUT
+}
+
+
+dig @8.8.8.8 +short www.google.com
+
+# DHCP
+ip addr flush dev eth0
+PID_FILE=/var/run/dhclient.pid
+if [ -f $PID_FILE ]; then
+    kill -9 $(cat $PID_FILE) || true
+    rm -f $PID_FILE
+fi
+dhclient -v eth0
+
+
+if [ -n "${options[oddservices]}" ]; then
+    echo Running services on non standard ports and open default ports 
+
+    echo Starting FTP 21514 and open default 20,21
+    nc -nvlt -p 20 &
+    nc -nvlt -p 21 &
+    (while true; do echo -e "220 ProFTPD 1.3.5e Server (Debian) $(hostname)" | nc -l -w 1 21514; done) &
+
+    echo Starting SMTP 1256 and open default 25, 465, 587
+    nc -nvlt -p 25 &
+    nc -nvlt -p 465 &
+    nc -nvlt -p 587 &
+    (while true; do echo -e "220 $(hostname) ESMTP Postfix (Ubuntu)" | nc -l -w 1 1256; done) &
+
+    echo Starting IMAP 5361 and open default ports 143, 993
+    nc -nvlt -p 143 &
+    nc -nvlt -p 993 &
+    (while true; do echo -e "* OK [CAPABILITY IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE STARTTLS AUTH=PLAIN] Dovecot (Ubuntu) ready.\r\n" \
+        | nc -l -w 1 5361; done) &
+
+    echo Starting POP3 23451 and open default 110, 995
+    nc -nvlt -p 110 &
+    nc -nvlt -p 995 &
+    (while true; do echo -ne "+OK POP3 Server ready\r\n" | nc -l -w 1 23451; done) &
+
+    echo starting TFTP UDP 69
+    (while true; do echo -ne "\0\x05\0\0\x07\0" | nc -u -l -w 1 69; done) &
+
+fi
+
+if [ -n "${options[snmp]}" ]; then
+    echo starting mock none snmpv3 on port UDP 161
+    (while true; do echo -ne " \x02\x01\ " | nc -u -l -w 1 161; done) &
+fi
+
+if [ -n "${options[snmpv3]}" ]; then
+    echo starting mock SNMPv3 UDP 161
+    (while true; do echo -ne "  \x02\x01\x030 \x02\x02Ji\x02    \x04\x01 \x02\x01\x03\x04" | nc -u -l -w 1 161; done) &
+fi
+
+if [ -n "${options[ssh]}" ]; then
+    echo Starting SSH server
+    /usr/local/sbin/sshd
+elif [ -n "${options[sshv1]}" ]; then
+    echo Starting SSHv1 server
+    echo 'Protocol 1' >> /usr/local/etc/sshd_config
+    /usr/local/sbin/sshd
+fi
+
+tail -f /dev/null

testing/example/mac1/results.json

@@ -0,0 +1,252 @@
+{
+    "device": {
+      "mac_addr": "7e:41:12:d2:35:6a"
+    },
+    "dns": {
+      "results": [
+        {
+          "name": "dns.network.from_device",
+          "description": "Verify the device sends DNS requests",
+          "expected_behavior": "The device sends DNS requests.",
+          "start": "2023-07-03T13:35:48.990574",
+          "result": "compliant",
+          "end": "2023-07-03T13:35:49.035528",
+          "duration": "0:00:00.044954"
+        },
+        {
+          "name": "dns.network.from_dhcp",
+          "description": "Verify the device allows for a DNS server to be entered automatically",
+          "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server",
+          "start": "2023-07-03T13:35:49.035701",
+          "result": "non-compliant",
+          "end": "2023-07-03T13:35:49.041532",
+          "duration": "0:00:00.005831"
+        },
+        {
+          "name": "dns.mdns",
+          "description": "If the device has MDNS (or any kind of IP multicast), can it be disabled",
+          "start": "2023-07-03T13:35:49.041679",
+          "result": "non-compliant",
+          "end": "2023-07-03T13:35:49.057430",
+          "duration": "0:00:00.015751"
+        }
+      ]
+    },
+    "nmap": {
+      "results": [
+        {
+          "name": "security.nmap.ports",
+          "description": "Run an nmap scan of open ports",
+          "expected_behavior": "Report all open ports",
+          "config": {
+            "security.services.ftp": {
+              "tcp_ports": {
+                "20": {
+                  "allowed": false,
+                  "description": "File Transfer Protocol (FTP) Server Data Transfer",
+                  "result": "compliant"
+                },
+                "21": {
+                  "allowed": false,
+                  "description": "File Transfer Protocol (FTP) Server Data Transfer",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check FTP port 20/21 is disabled and FTP is not running on any port",
+              "expected_behavior": "There is no FTP service running on any port"
+            },
+            "security.services.ssh": {
+              "tcp_ports": {
+                "22": {
+                  "allowed": true,
+                  "description": "Secure Shell (SSH) server",
+                  "version": "2.0",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check TELNET port 23 is disabled and TELNET is not running on any port",
+              "expected_behavior": "There is no FTP service running on any port"
+            },
+            "security.services.telnet": {
+              "tcp_ports": {
+                "23": {
+                  "allowed": false,
+                  "description": "Telnet Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check TELNET port 23 is disabled and TELNET is not running on any port",
+              "expected_behavior": "There is no FTP service running on any port"
+            },
+            "security.services.smtp": {
+              "tcp_ports": {
+                "25": {
+                  "allowed": false,
+                  "description": "Simple Mail Transfer Protocol (SMTP) Server",
+                  "result": "compliant"
+                },
+                "465": {
+                  "allowed": false,
+                  "description": "Simple Mail Transfer Protocol over SSL (SMTPS) Server",
+                  "result": "compliant"
+                },
+                "587": {
+                  "allowed": false,
+                  "description": "Simple Mail Transfer Protocol via TLS (SMTPS) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check SMTP port 25 is disabled and ports 465 or 587 with SSL encryption are (not?) enabled and SMTP is not running on any port.",
+              "expected_behavior": "There is no smtp service running on any port"
+            },
+            "security.services.http": {
+              "tcp_ports": {
+                "80": {
+                  "service_scan": {
+                    "script": "http-methods"
+                  },
+                  "allowed": false,
+                  "description": "Administrative Insecure Web-Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check that there is no HTTP server running on any port",
+              "expected_behavior": "Device is unreachable on port 80 (or any other port) and only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)"
+            },
+            "security.services.pop": {
+              "tcp_ports": {
+                "110": {
+                  "allowed": false,
+                  "description": "Post Office Protocol v3 (POP3) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check POP port 110 is disalbed and POP is not running on any port",
+              "expected_behavior": "There is no pop service running on any port"
+            },
+            "security.services.imap": {
+              "tcp_ports": {
+                "143": {
+                  "allowed": false,
+                  "description": "Internet Message Access Protocol (IMAP) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check IMAP port 143 is disabled and IMAP is not running on any port",
+              "expected_behavior": "There is no imap service running on any port"
+            },
+            "security.services.snmpv3": {
+              "tcp_ports": {
+                "161": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP)",
+                  "result": "compliant"
+                },
+                "162": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP) Trap",
+                  "result": "compliant"
+                }
+              },
+              "udp_ports": {
+                "161": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP)"
+                },
+                "162": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP) Trap"
+                }
+              },
+              "description": "Check SNMP port 161/162 is disabled.  If SNMP is an essential service, check it supports version 3",
+              "expected_behavior": "Device is unreachable on port 161 (or any other port) and device is unreachable on port 162 (or any other port) unless SNMP is essential in which case it is SNMPv3 is used."
+            },
+            "security.services.https": {
+              "tcp_ports": {
+                "80": {
+                  "allowed": false,
+                  "description": "Administrative Secure Web-Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check that if there is a web server running it is running on a secure port.",
+              "expected_behavior": "Device only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)"
+            },
+            "security.services.vnc": {
+              "tcp_ports": {
+                "5800": {
+                  "allowed": false,
+                  "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol Over HTTP",
+                  "result": "compliant"
+                },
+                "5500": {
+                  "allowed": false,
+                  "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check VNC is disabled on any port",
+              "expected_behavior": "Device cannot be accessed /connected to via VNc on any port"
+            },
+            "security.services.tftp": {
+              "udp_ports": {
+                "69": {
+                  "allowed": false,
+                  "description": "Trivial File Transfer Protocol (TFTP) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check TFTP port 69 is disabled (UDP)",
+              "expected_behavior": "There is no tftp service running on any port"
+            },
+            "security.services.ntp": {
+              "udp_ports": {
+                "123": {
+                  "allowed": false,
+                  "description": "Network Time Protocol (NTP) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check NTP port 123 is disabled and the device is not operating as an NTP server",
+              "expected_behavior": "The device dos not respond to NTP requests when it's IP is set as the NTP server on another device"
+            }
+          },
+          "start": "2023-07-03T13:36:26.923704",
+          "result": "compliant",
+          "end": "2023-07-03T13:36:52.965535",
+          "duration": "0:00:26.041831"
+        }
+      ]
+    },
+    "baseline": {
+      "results": [
+        {
+          "name": "baseline.pass",
+          "description": "Simulate a compliant test",
+          "expected_behavior": "A compliant test result is generated",
+          "start": "2023-07-03T13:37:29.100681",
+          "result": "compliant",
+          "end": "2023-07-03T13:37:29.100869",
+          "duration": "0:00:00.000188"
+        },
+        {
+          "name": "baseline.fail",
+          "description": "Simulate a non-compliant test",
+          "expected_behavior": "A non-compliant test result is generated",
+          "start": "2023-07-03T13:37:29.100961",
+          "result": "non-compliant",
+          "end": "2023-07-03T13:37:29.101089",
+          "duration": "0:00:00.000128"
+        },
+        {
+          "name": "baseline.skip",
+          "description": "Simulate a skipped test",
+          "expected_behavior": "A skipped test result is generated",
+          "start": "2023-07-03T13:37:29.101164",
+          "result": "skipped",
+          "end": "2023-07-03T13:37:29.101283",
+          "duration": "0:00:00.000119"
+        }
+      ]
+    }
+  }