Keep test results in memory

framework/python/src/api/api.py

@@ -108,11 +108,14 @@ async def start_test_run(self, request: Request, response: Response):
       response.status_code = status.HTTP_400_BAD_REQUEST
       return self._generate_msg(False, "Invalid JSON received")
 
-    if "device" not in body_json or "mac_addr" not in body_json["device"]:
+    if "device" not in body_json or not (
+      "mac_addr" in body_json["device"] and
+      "firmware" in body_json["device"]):
       response.status_code = status.HTTP_400_BAD_REQUEST
       return self._generate_msg(False, "Invalid request received")
 
     device = self._session.get_device(body_json["device"]["mac_addr"])
+    device.firmware = body_json["device"]["firmware"]
 
     # Check Test Run is not already running
     if self._test_run.get_session().get_status() != "Idle":
@@ -123,12 +126,13 @@ async def start_test_run(self, request: Request, response: Response):
     # Check if requested device is known in the device repository
     if device is None:
       response.status_code = status.HTTP_404_NOT_FOUND
-      return self._generate_msg(False, "A device with that MAC address could not be found")
+      return self._generate_msg(False,
+                                "A device with that MAC address could not be found")
 
     # Check Test Run is able to start
     if self._test_run.get_net_orc().check_config() is False:
       response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
-      return self._generate_msg(False, "Configured interfaces are not ready for use. Ensure both interfaces are connected.")
+      return self._generate_msg(False,"Configured interfaces are not ready for use. Ensure required interfaces are connected.")
 
     self._test_run.get_session().set_target_device(device)
     LOGGER.info(f"Starting Test Run with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}")

framework/python/src/common/device.py

@@ -25,5 +25,15 @@ class Device():
   model: str = None
   test_modules: str = None
   ip_addr: str = None
+  firmware: str = None
   device_folder: str = None
   max_device_reports: int = None
+
+  def to_json(self):
+    device_json = {}
+    device_json['mac_addr'] = self.mac_addr
+    device_json['manufacturer'] = self.manufacturer
+    device_json['model'] = self.model
+    if self.firmware is not None:
+      device_json['firmware'] = self.firmware
+    return device_json

framework/python/src/common/session.py

@@ -35,7 +35,8 @@ def __init__(self, config_file):
     self._device = None
     self._started = None
     self._finished = None
-    self._tests = []
+    self._results = []
+    self._runtime_params = []
 
     self._config_file = config_file
 
@@ -54,6 +55,9 @@ def get_started(self):
   def get_finished(self):
     return self._finished
 
+  def stop(self):
+    self._finished = datetime.datetime.now()
+
   def _get_default_config(self):
     return {
       'network': {
@@ -110,6 +114,12 @@ def get_runtime(self):
   def get_log_level(self):
     return self._config.get(LOG_LEVEL_KEY)
 
+  def get_runtime_params(self):
+    return self._runtime_params
+
+  def add_runtime_param(self, param):
+    self._runtime_params.append(param)
+
   def get_device_interface(self):
     return self._config.get(NETWORK_KEY, {}).get(DEVICE_INTF_KEY)
 
@@ -157,13 +167,16 @@ def get_status(self):
   def set_status(self, status):
     self._status = status
 
-  def get_tests(self):
-    return self._tests
+  def get_test_results(self):
+    return self._results
+
+  def add_test_result(self, test_result):
+    self._results.append(test_result)
 
   def reset(self):
     self.set_status('Idle')
     self.set_target_device(None)
-    self._tests = []
+    self._results = []
     self._started = None
     self._finished = None
 
@@ -173,5 +186,5 @@ def to_json(self):
       'device': self.get_target_device(),
       'started': self.get_started(),
       'finished': self.get_finished(),
-      'tests': self.get_tests()
+      'results': self.get_test_results()
     }

framework/python/src/core/testrun.py

@@ -81,7 +81,14 @@ def __init__(self,
     # Catch any exit signals
     self._register_exits()
 
+    # Create session
     self._session = TestRunSession(config_file=self._config_file)
+
+    if single_intf:
+      self._session.add_runtime_param('single_intf')
+    if net_only:
+      self._session.add_runtime_param('net_only')
+
     self._load_all_devices()
 
     self._net_orc = net_orc.NetworkOrchestrator(
@@ -93,6 +100,12 @@ def __init__(self,
       self._net_orc)
 
     if self._no_ui:
+      # Check Test Run is able to start
+      if self.get_net_orc().check_config() is False:
+        return
+
+      # Any additional checks that need to be performed go here
+
       self.start()
     else:
       self._api = Api(self)
@@ -192,7 +205,7 @@ def start(self):
           self.get_net_orc().monitor_in_progress()):
           time.sleep(5)
 
-      self.stop()
+    self.stop()
 
   def stop(self, kill=False):
     self._set_status('Stopping')
@@ -254,27 +267,26 @@ def get_device(self, mac_addr):
 
   def _device_discovered(self, mac_addr):
 
-    if self.get_session().get_target_device() is not None:
-      if mac_addr != self.get_session().get_target_device().mac_addr:
-        # Ignore discovered device
-        return
+    device = self.get_session().get_target_device()
 
-    self._set_status('Identifying device')
-    device = self.get_device(mac_addr)
     if device is not None:
-      LOGGER.info(
-        f'Discovered {device.manufacturer} {device.model} on the network')
+      if mac_addr != device.mac_addr:
+        # Ignore discovered device because it is not the target device
+        return
     else:
-      device = Device(mac_addr=mac_addr)
-      self._devices.append(device)
-      LOGGER.info(
-        f'A new device has been discovered with mac address {mac_addr}')
+      device = self.get_device(mac_addr)
+      if device is None:
+        return
+
+      self.get_session().set_target_device(device)
+
+    LOGGER.info(
+        f'Discovered {device.manufacturer} {device.model} on the network')
 
   def _device_stable(self, mac_addr):
-    device = self.get_device(mac_addr)
     LOGGER.info(f'Device with mac address {mac_addr} is ready for testing.')
     self._set_status('In progress')
-    self._test_orc.run_test_modules(device)
+    self._test_orc.run_test_modules()
     self._set_status('Complete')
 
   def _set_status(self, status):

framework/python/src/net_orc/network_orchestrator.py

@@ -90,11 +90,30 @@ def start(self):
 
   def check_config(self):
 
-    if not util.interface_exists(self._session.get_internet_interface()) or not util.interface_exists(
-        self._session.get_device_interface()):
-      LOGGER.error('Configured interfaces are not ready for use. ' +
-                   'Ensure both interfaces are connected.')
-      return False
+    device_interface_ready = util.interface_exists(
+        self._session.get_device_interface())
+    internet_interface_ready = util.interface_exists(
+        self._session.get_internet_interface())
+
+    if 'single_intf' in self._session.get_runtime_params():
+      # Check for device interface only
+      if not device_interface_ready:
+        LOGGER.error('Device interface is not ready for use. ' +
+                     'Ensure device interface is connected.')
+        return False
+    else:
+      if not device_interface_ready and not internet_interface_ready:
+        LOGGER.error('Both device and internet interfaces are not ready for use. ' +
+                     'Ensure both interfaces are connected.')
+        return False
+      elif not device_interface_ready:
+        LOGGER.error('Device interface is not ready for use. ' +
+                     'Ensure device interface is connected.')
+        return False
+      elif not internet_interface_ready:
+        LOGGER.error('Internet interface is not ready for use. ' +
+                     'Ensure internet interface is connected.')
+        return False
     return True
 
   def start_network(self):
@@ -310,20 +329,19 @@ def _ci_post_network_create(self):
   def create_net(self):
     LOGGER.info('Creating baseline network')
 
-    if self._single_intf:
-      self._ci_pre_network_create()
-
-    # Remove IP from internet adapter
-    util.run_command('ifconfig ' + self._session.get_internet_interface() + ' 0.0.0.0')
+    # TODO: This is not just for CI
+    #if self._single_intf:
+      #self._ci_pre_network_create()
 
     # Setup the virtual network
     if not self._ovs.create_baseline_net(verify=True):
       LOGGER.error('Baseline network validation failed.')
       self.stop()
       sys.exit(1)
 
-    if self._single_intf:
-      self._ci_post_network_create()
+    # TODO: This is not just for CI
+    #if self._single_intf:
+      #self._ci_post_network_create()
 
     self._create_private_net()
 

framework/python/src/net_orc/ovs_control.py

@@ -76,13 +76,17 @@ def validate_baseline_network(self):
     # Verify the OVS setup of the virtual network
     LOGGER.debug('Validating baseline network')
 
+    dev_bridge = True
+    int_bridge = True
+
     # Verify the device bridge
     dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._session.get_device_interface()])
     LOGGER.debug('Device bridge verified: ' + str(dev_bridge))
 
     # Verify the internet bridge
-    int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()])
-    LOGGER.debug('Internet bridge verified: ' + str(int_bridge))
+    if 'single_intf' not in self._session.get_runtime_params():
+      int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._session.get_internet_interface()])
+      LOGGER.debug('Internet bridge verified: ' + str(int_bridge))
 
     return dev_bridge and int_bridge
 
@@ -103,21 +107,19 @@ def verify_bridge(self, bridge_name, ports):
   def create_baseline_net(self, verify=True):
     LOGGER.debug('Creating baseline network')
 
-    # Remove IP from internet adapter
-    self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0')
-
     # Create data plane
     self.add_bridge(DEVICE_BRIDGE)
 
     # Create control plane
     self.add_bridge(INTERNET_BRIDGE)
 
-    # Remove IP from internet adapter
-    self.set_interface_ip(self._session.get_internet_interface(), '0.0.0.0')
-
     # Add external interfaces to data and control plane
     self.add_port(self._session.get_device_interface(), DEVICE_BRIDGE)
-    self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE)
+
+    # Remove IP from internet adapter
+    if not 'single_intf' in self._session.get_runtime_params():
+      self.set_interface_ip(interface=self._session.get_internet_interface(), ip_addr='0.0.0.0')
+      self.add_port(self._session.get_internet_interface(), INTERNET_BRIDGE)
 
     # Enable forwarding of eapol packets
     self.add_flow(bridge_name=DEVICE_BRIDGE,