Pass log level to test module

.gitignore

@@ -12,6 +12,7 @@ testing/unit/dns/output/
 testing/unit/nmap/output/
 testing/unit/ntp/output/
 testing/unit/tls/output/
+testing/unit/tls/tmp/
 testing/unit/report/output/
 
 *.deb

cmd/package

@@ -17,6 +17,10 @@
 # Creates a package for Testrun
 
 MAKE_SRC_DIR=make
+TESTRUN_VER="1-2-2"
+
+# Delete existing make files
+rm -rf $MAKE_SRC_DIR/usr
 
 # Delete existing make files
 rm -rf $MAKE_SRC_DIR/usr
@@ -56,4 +60,7 @@ cp -r {framework,modules} $MAKE_SRC_DIR/usr/local/testrun
 dpkg-deb --build --root-owner-group make
 
 # Rename the .deb file
-mv make.deb testrun_1-2-1_amd64.deb
+mv make.deb testrun_${TESTRUN_VER}_amd64.deb
+
+# Echo package version
+echo Created installation package at testrun_${TESTRUN_VER}_amd64.deb

docs/get_started.md

@@ -109,5 +109,8 @@ If you encounter any issues or need assistance, consider the following:
 - Check the configuration settings.
 - Refer to the Testrun documentation or ask for assistance in the issues page: https://github.com/google/testrun/issues
 
+# Reviewing
+Once you have completed a test attempt, you may want to review the test report provided by Testrun. For more information about what Testrun looks for when testing, and what the output means, take a look at the testing documentation: [Testing](/docs/test/index.md).
+
 # Uninstall
-To uninstall Testrun, use the built-in dpkg uninstall command to remove Testrun correctly. For Testrun, this would be:  ```sudo apt-get remove testrun```
+To uninstall Testrun, use the built-in dpkg uninstall command to remove Testrun correctly. For Testrun, this would be:  ```sudo apt-get remove testrun```. Note that this 

docs/test/index.md

@@ -0,0 +1,5 @@
+# Testing
+
+The test requirements that are investigated by Testrun can be found in the [test modules documentation](/docs/test/modules.md).
+
+To understand the testing results, various definitions of test results and requirements are specified in the [statuses documentation](/docs/test/statuses.md).

docs/test/statuses.md

@@ -0,0 +1,32 @@
+# Test Statuses
+Testrun will output the result and description of each automated test. The test results will be one of the following:
+
+| Name | Description | What next? |
+|---|---|---|
+| Compliant | The device implements the required feature correctly | Nothing |
+| Non-Compliant | The device does not support the specified requirements for the test | Modify or implement the required functionality on the device |
+| Feature Not Present | The device does not implement a feature covered by the test | You may implement the  functionality (not required) |
+| Error | An error occured whilst running the test | Create a bug report requesting additional support to diagnose the issue |
+| Skipped | The test has not been executed because a linked test did not produce a compliant result | You may implement the functionality (not required) |
+
+## Test requirement
+Testrun also determines whether each test is required for the device to receive an overall compliant result. These rules are:
+
+| Name | Description |
+|---|---|
+| Required | The device must implement the feature |
+| Recommended | The device should implement the feature, but will not receive an overall Non-Compliant if not implemented |
+| Roadmap | The device should implement this feature in the future, but is not required at the moment |
+| Required If Applicable | If the device implements this feature, it must be implemented correctly (as per the test requirements) |
+
+## Testrun Statuses
+Once testing is completed, an overall result for the test attempt will be produced. This is calculated by comparing the result of all tests, and whether they are required or not required.
+
+### Compliant
+All required tests are implemented correctly, and all required if applicable tests are implemented correctly (where the feature has been implemented).
+
+### Non-Compliant
+One or more of the required tests (or required if applicable tests) have produced a non-compliant result.
+
+### Error
+One of more of the required tests (or required if applicable tests) have not executed correctly. This does not necessarily indicate that the device is compliant or non-compliant.

framework/python/src/api/api.py

@@ -21,6 +21,7 @@
 import os
 import psutil
 import requests
+import signal
 import threading
 import uvicorn
 from urllib.parse import urlparse
@@ -46,7 +47,7 @@ class Api:
   def __init__(self, test_run):
 
     self._test_run = test_run
-    self._name = "TestRun API"
+    self._name = "Testrun API"
     self._router = APIRouter()
 
     self._session = self._test_run.get_session()
@@ -66,6 +67,9 @@ def __init__(self, test_run):
                                methods=["POST"])
     self._router.add_api_route("/system/status",
                                self.get_status)
+    self._router.add_api_route("/system/shutdown",
+                               self.shutdown,
+                               methods=["POST"])
 
     self._router.add_api_route("/system/version",
                                self.get_version)
@@ -243,8 +247,35 @@ async def stop_test_run(self):
   async def get_status(self):
     return self._test_run.get_session().to_json()
 
+  def shutdown(self, response: Response):
+
+    LOGGER.debug("Received request to shutdown Testrun")
+
+    # Check that Testrun is not currently running
+    if (self._session.get_status() not in [
+            "Cancelled",
+            "Compliant",
+            "Non-Compliant",
+            "Idle"]):
+      LOGGER.debug("Unable to shutdown Testrun as Testrun is in progress")
+      response.status_code = 400
+      return self._generate_msg(
+        False,
+        "Unable to shutdown. A test is currently in progress."
+      )
+
+    self._test_run.shutdown()
+    os.kill(os.getpid(), signal.SIGTERM)
+
   async def get_version(self, response: Response):
+
+    # Add defaults
     json_response = {}
+    json_response["installed_version"] = "v" + self._test_run.get_version()
+    json_response["update_available"] = False
+    json_response["latest_version"] = None
+    json_response["latest_version_url"] = (
+      "https://github.com/google/testrun/releases")
 
     # Obtain the current version
     current_version = self._session.get_version()
@@ -264,9 +295,9 @@ async def get_version(self, response: Response):
       # Check OK response was received
       if version_check.status_code != 200:
         response.status_code = 500
-        LOGGER.error(version_check.content)
-        return self._generate_msg(False, "Failed to fetch latest version. " +
-                                  "Please, check the internet connection")
+        LOGGER.debug(version_check.content)
+        LOGGER.error("Failed to fetch latest version")
+        return json_response
 
       # Extract version number from response, removing the leading 'v'
       latest_version_no = version_check.json()["name"].strip("v")
@@ -289,8 +320,7 @@ async def get_version(self, response: Response):
       response.status_code = 500
       LOGGER.error("Failed to fetch latest version")
       LOGGER.debug(e)
-      return self._generate_msg(False, "Failed to fetch latest version. " +
-                                "Please, check the internet connection")
+      return json_response
 
   async def get_reports(self, request: Request):
     LOGGER.debug("Received reports list request")
@@ -304,7 +334,7 @@ async def get_reports(self, request: Request):
     for report in reports:
       # report URL is currently hard coded as localhost so we can
       # replace that to fix the IP dynamically from the requester
-      report["report"] = report["report"].replace("localhost",server_ip)
+      report["report"] = report["report"].replace("localhost", server_ip)
     return reports
 
   async def delete_report(self, request: Request, response: Response):
@@ -531,8 +561,8 @@ def _validate_device_json(self, json_obj):
       return False
 
     # Check length of strings
-    if len(json_obj.get(DEVICE_MANUFACTURER_KEY)) > 64 or len(
-      json_obj.get(DEVICE_MODEL_KEY)) > 64:
+    if len(json_obj.get(DEVICE_MANUFACTURER_KEY)) > 28 or len(
+      json_obj.get(DEVICE_MODEL_KEY)) > 28:
       return False
 
     disallowed_chars = ["/", "\\", "\'", "\"", ";"]

framework/python/src/common/device.py

@@ -41,6 +41,9 @@ def add_report(self, report):
   def get_reports(self):
     return self.reports
 
+  def clear_reports(self):
+    self.reports = []
+
   def remove_report(self, timestamp: datetime):
     for report in self.reports:
       if report.get_started().strftime('%Y-%m-%dT%H:%M:%S') == timestamp:

framework/python/src/common/session.py

@@ -31,10 +31,10 @@
 
 LOGGER = logger.get_logger('session')
 
-class TestRunSession():
+class TestrunSession():
   """Represents the current session of Test Run."""
 
-  def __init__(self, config_file):
+  def __init__(self, config_file, version):
     self._status = 'Idle'
     self._device = None
     self._started = None
@@ -46,8 +46,7 @@ def __init__(self, config_file):
     self._total_tests = 0
     self._report_url = None
 
-    self._version = None
-    self._load_version()
+    self._load_version(default_version=version)
 
     self._config_file = config_file
     self._config = self._get_default_config()
@@ -90,7 +89,7 @@ def _get_default_config(self):
       'log_level': 'INFO',
       'startup_timeout': 60,
       'monitor_period': 30,
-      'max_device_reports': 5,
+      'max_device_reports': 0,
       'api_url': 'http://localhost',
       'api_port': 8000
     }
@@ -141,14 +140,18 @@ def _load_config(self):
 
       LOGGER.debug(self._config)
 
-  def _load_version(self):
+  def _load_version(self, default_version):
     version_cmd = util.run_command(
       'dpkg-query --showformat=\'${Version}\' --show testrun')
-
-    if version_cmd:
+    # index 1 of response is the stderr byte stream so if
+    # it has any data in it, there was an error and we 
+    # did not resolve the version and we'll use the fallback
+    if len(version_cmd[1]) == 0:
       version = version_cmd[0]
-      LOGGER.info(f'Running Testrun version {version}')
-    self._version = version
+      self._version = version
+    else:
+      self._version = default_version
+    LOGGER.info(f'Running Testrun version {self._version}')
 
   def get_version(self):
     return self._version
@@ -294,6 +297,7 @@ def reset(self):
     self.set_target_device(None)
     self._report_url = None
     self._total_tests = 0
+    self._module_reports = []
     self._results = []
     self._started = None
     self._finished = None