Test api

.github/workflows/testing.yml

@@ -1,7 +1,6 @@
 name: Testrun test suite
 
 on:
-  push:
   pull_request:
   schedule:
     - cron: '0 13 * * *'
@@ -21,14 +20,34 @@ jobs:
   testrun_tests:
     name:  Tests
     runs-on: ubuntu-20.04
-    needs: testrun_baseline
     timeout-minutes: 40
     steps:
       - name: Checkout source
         uses: actions/checkout@v2.3.4
       - name: Run tests
         shell: bash {0}
         run: testing/tests/test_tests
+      - name: Archive runtime results
+        if: ${{ always() }}
+        run: sudo tar --exclude-vcs -czf runtime.tgz runtime/
+      - name: Upload runtime results
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          if-no-files-found: error
+          name: runtime_${{ github.workflow }}_${{ github.run_id }}
+          path: runtime.tgz
+
+  testrun_api:
+    name:  API
+    runs-on: ubuntu-20.04
+    timeout-minutes: 40
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.4
+      - name: Run tests
+        shell: bash {0}
+        run: testing/api/test_api
 
   pylint:
     name:  Pylint

bin/testrun

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 if [[ "$EUID" -ne 0 ]]; then
-    echo "Must run as root. Use sudo testrun"
+    echo "Must run as root. Use sudo $0"
     exit 1
 fi
 

cmd/install

@@ -24,4 +24,7 @@ pip3 install -r framework/requirements.txt
 # required by python package weasyprint
 sudo apt-get install libpangocairo-1.0-0
 
+#TODO move into docker build process
+(cd modules/ui && npm install && npm run build)
+
 deactivate

framework/python/src/common/session.py

@@ -87,17 +87,21 @@ def _load_config(self):
       if (NETWORK_KEY in config_file_json
           and DEVICE_INTF_KEY in config_file_json.get(NETWORK_KEY)
           and INTERNET_INTF_KEY in config_file_json.get(NETWORK_KEY)):
-        self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY)
-        self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(NETWORK_KEY, {}).get(INTERNET_INTF_KEY)
+        self._config[NETWORK_KEY][DEVICE_INTF_KEY] = config_file_json.get(
+          NETWORK_KEY, {}).get(DEVICE_INTF_KEY)
+        self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(
+          NETWORK_KEY, {}).get(INTERNET_INTF_KEY)
 
       if RUNTIME_KEY in config_file_json:
         self._config[RUNTIME_KEY] = config_file_json.get(RUNTIME_KEY)
 
       if STARTUP_TIMEOUT_KEY in config_file_json:
-        self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(STARTUP_TIMEOUT_KEY)
+        self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(
+          STARTUP_TIMEOUT_KEY)
 
       if MONITOR_PERIOD_KEY in config_file_json:
-        self._config[MONITOR_PERIOD_KEY] = config_file_json.get(MONITOR_PERIOD_KEY)
+        self._config[MONITOR_PERIOD_KEY] = config_file_json.get(
+          MONITOR_PERIOD_KEY)
 
       if LOG_LEVEL_KEY in config_file_json:
         self._config[LOG_LEVEL_KEY] = config_file_json.get(LOG_LEVEL_KEY)
@@ -106,7 +110,8 @@ def _load_config(self):
         self._config[API_PORT_KEY] = config_file_json.get(API_PORT_KEY)
 
       if MAX_DEVICE_REPORTS_KEY in config_file_json:
-        self._config[MAX_DEVICE_REPORTS_KEY] = config_file_json.get(MAX_DEVICE_REPORTS_KEY)
+        self._config[MAX_DEVICE_REPORTS_KEY] = config_file_json.get(
+          MAX_DEVICE_REPORTS_KEY)
 
   def _save_config(self):
     with open(self._config_file, 'w', encoding='utf-8') as f:

framework/python/src/common/testreport.py

@@ -94,7 +94,7 @@ def to_pdf(self):
     pdf_bytes = BytesIO()
     HTML(string=report_html).write_pdf(pdf_bytes)
     return pdf_bytes
-   
+
   def to_html(self):
     json_data = self.to_json()
     return f'''
@@ -118,18 +118,19 @@ def to_html(self):
     '''
 
   def generate_test_sections(self,json_data):
-    results = json_data["tests"]["results"]
-    sections = ""
+    results = json_data['tests']['results']
+    sections = ''
     for result in results:
-        sections += self.generate_test_section(result)
+      sections += self.generate_test_section(result)
     return sections
 
   def generate_test_section(self, result):
     section_content = '<section class="test-section">\n'
     for key, value in result.items():
-        if value is not None:  # Check if the value is not None
-            formatted_key = key.replace('_', ' ').title()  # Replace underscores and capitalize
-            section_content += f'<p><strong>{formatted_key}:</strong> {value}</p>\n'
+      if value is not None:  # Check if the value is not None
+        # Replace underscores and capitalize
+        formatted_key = key.replace('_', ' ').title()
+        section_content += f'<p><strong>{formatted_key}:</strong> {value}</p>\n'
     section_content += '</section>\n<div style="margin-bottom: 40px;"></div>\n'
     return section_content
 
@@ -171,4 +172,4 @@ def generate_css(self):
       text-decoration: none;
       color: #007bff;
     }
-    '''
+    '''

framework/requirements.txt

@@ -14,4 +14,8 @@ weasyprint
 fastapi==0.99.1
 psutil
 uvicorn
-pydantic==1.10.11
+pydantic==1.10.11
+
+# Requirements for testing
+pytest
+pytest-timeout

modules/network/dhcp-1/dhcp-1.Dockerfile

@@ -25,7 +25,7 @@ RUN apt-get install -y wget
 RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/
 
 # Install dhcp server
-RUN apt-get install -y isc-dhcp-server radvd systemd
+RUN apt-get update && apt-get install -y isc-dhcp-server radvd systemd
 
 # Copy over all configuration files
 COPY $MODULE_DIR/conf /testrun/conf

modules/network/dhcp-2/dhcp-2.Dockerfile

@@ -25,7 +25,7 @@ RUN apt-get install -y wget
 RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/
 
 # Install dhcp server
-RUN apt-get install -y isc-dhcp-server radvd systemd
+RUN apt-get update && apt-get install -y isc-dhcp-server radvd systemd
 
 # Copy over all configuration files
 COPY $MODULE_DIR/conf /testrun/conf

modules/test/baseline/python/src/baseline_module.py

@@ -27,17 +27,17 @@ def __init__(self, module):
     global LOGGER
     LOGGER = self._get_logger()
 
-  def _baseline_pass(self):
+  def _baseline_compliant(self):
     LOGGER.info('Running baseline pass test')
     LOGGER.info('Baseline pass test finished')
     return True, 'Baseline pass test ran successfully'
 
-  def _baseline_fail(self):
+  def _baseline_non_compliant(self):
     LOGGER.info('Running baseline fail test')
     LOGGER.info('Baseline fail test finished')
     return False, 'Baseline fail test ran successfully'
 
-  def _baseline_skip(self):
+  def _baseline_informational(self):
     LOGGER.info('Running baseline skip test')
     LOGGER.info('Baseline skip test finished')
     return None, 'Baseline skip test ran successfully'

testing/api/mockito/get_devices.json

@@ -0,0 +1,46 @@
+[
+    {
+      "mac_addr": "00:1e:42:35:73:c4",
+      "manufacturer": "Teltonika",
+      "model": "TRB 140",
+      "test_modules": {
+        "dns": {
+          "enabled": false
+        },
+        "connection": {
+          "enabled": true
+        },
+        "ntp": {
+          "enabled": false
+        },
+        "baseline": {
+          "enabled": false
+        },
+        "nmap": {
+          "enabled": false
+        }
+      }
+    },
+    {
+      "mac_addr": "aa:bb:cc:dd:ee:ff",
+      "manufacturer": "Manufacturer X",
+      "model": "Device X",
+      "test_modules": {
+        "dns": {
+          "enabled": true
+        },
+        "connection": {
+          "enabled": true
+        },
+        "ntp": {
+          "enabled": true
+        },
+        "baseline": {
+          "enabled": false
+        },
+        "nmap": {
+          "enabled": true
+        }
+      }
+    }
+  ]

testing/api/mockito/invalid_request.json

@@ -0,0 +1,3 @@
+{
+    "error": "Invalid request received"
+}

testing/api/mockito/running_system_status.json

@@ -0,0 +1,26 @@
+{
+    "status": "In Progress",
+    "device": {
+      "manufacturer": "Delta",
+      "model": "03-DIN-CPU",
+      "mac_addr": "01:02:03:04:05:06",
+      "firmware": "1.2.2"
+    },
+    "started": "2023-06-22T09:20:00.123Z",
+    "finished": null,
+    "tests": {
+      "total": 26,
+      "results": [
+        {
+          "name": "dns.network.hostname_resolution",
+          "description": "The device should resolve hostnames",
+          "result": "Compliant"
+        },
+        {
+          "name": "dns.network.from_dhcp",
+          "description": "The device should use the DNS server provided by the DHCP server",
+          "result": "Non-Compliant"
+        } 
+      ]
+    }
+  }

testing/api/test_api

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ifconfig
+
+# Setup requirements
+sudo apt-get update
+sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils isc-dhcp-client
+
+pip3 install pytest
+
+# Setup device network
+sudo ip link add dev dummynet type dummy
+sudo ip link add dev endev0a type veth peer name endev0b
+sudo ip link set dev endev0a up
+sudo ip link set dev endev0b up
+sudo docker network create -d macvlan -o parent=endev0b endev0
+
+sudo ip link add dev dummynet type dummy
+
+# Start OVS
+sudo /usr/share/openvswitch/scripts/ovs-ctl start
+
+# Build Test Container
+sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f  ./testing/docker/ci_test_device1/Dockerfile
+
+sudo chown -R $USER local
+
+cat <<EOF >local/system.json
+{
+  "network": {
+    "device_intf": "endev0a",
+    "internet_intf": "dummynet"
+  },
+  "log_level": "DEBUG"
+}
+EOF
+
+sudo cmd/install
+
+# Needs to be sudo because this invokes bin/testrun 
+sudo venv/bin/python -m pytest -v testing/api/test_api.py
+
+exit $?