Sync dev to main

cmd/start

@@ -22,25 +22,4 @@ source venv/bin/activate
 export PYTHONPATH="$PWD/framework/python/src"
 python -u framework/python/src/core/test_runner.py $@
 
-# TODO: Work in progress code for containerization of OVS module
-# asyncRun() {
-#     "$@" &
-#     pid="$!"
-#     echo "PID Running: " $pid
-#     trap "echo 'Stopping PID $pid'; kill -SIGTERM $pid" SIGINT SIGTERM
-
-#     sleep 10
-
-#     # A signal emitted while waiting will make the wait command return code > 128
-#     # Let's wrap it in a loop that doesn't end before the process is indeed stopped
-#     while kill -0 $pid > /dev/null 2>&1; do
-#     #while $(kill -0 $pid 2>/dev/null); do
-#         wait
-#     done
-# }
-
-# # -u flag allows python print statements
-# # to be logged by docker by running unbuffered
-# asyncRun python3 -u python/src/run.py $@
-
 deactivate

framework/python/src/common/logger.py

@@ -21,7 +21,7 @@
 _LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s'
 _DATE_FORMAT = '%b %02d %H:%M:%S'
 _DEFAULT_LEVEL = logging.INFO
-_CONF_DIR = 'conf'
+_CONF_DIR = 'local'
 _CONF_FILE_NAME = 'system.json'
 
 # Set log level

framework/python/src/common/util.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 """Provides basic utilities for the network orchestrator."""
+import getpass
+import os
 import subprocess
 import shlex
 from common import logger
@@ -37,7 +39,7 @@ def run_command(cmd, output=True):
 
   if process.returncode != 0 and output:
     err_msg = f'{stderr.strip()}. Code: {process.returncode}'
-    LOGGER.error('Command Failed: ' + cmd)
+    LOGGER.error('Command failed: ' + cmd)
     LOGGER.error('Error: ' + err_msg)
   else:
     success = True
@@ -50,6 +52,44 @@ def run_command(cmd, output=True):
 def interface_exists(interface):
   return interface in netifaces.interfaces()
 
-
 def prettify(mac_string):
   return ':'.join([f'{ord(b):02x}' for b in mac_string])
+
+def get_host_user():
+  user = get_os_user()
+
+  # If primary method failed, try secondary
+  if user is None:
+    user = get_user()
+
+  return user
+
+def get_os_user():
+  user = None
+  try:
+    user = os.getlogin()
+  except OSError:
+    # Handle the OSError exception
+    LOGGER.error('An OS error occured whilst calling os.getlogin()')
+  except Exception:
+    # Catch any other unexpected exceptions
+    LOGGER.error('An unknown exception occured whilst calling os.getlogin()')
+  return user
+
+def get_user():
+  user = None
+  try:
+    user = getpass.getuser()
+  except (KeyError, ImportError, ModuleNotFoundError, OSError) as e:
+    # Handle specific exceptions individually
+    if isinstance(e, KeyError):
+      LOGGER.error('USER environment variable not set or unavailable.')
+    elif isinstance(e, ImportError):
+      LOGGER.error('Unable to import the getpass module.')
+    elif isinstance(e, ModuleNotFoundError):
+      LOGGER.error('The getpass module was not found.')
+    elif isinstance(e, OSError):
+      LOGGER.error('An OS error occurred while retrieving the username.')
+    else:
+      LOGGER.error('An exception occurred:', e)
+  return user

framework/python/src/core/device.py

@@ -22,6 +22,6 @@
 class Device(NetworkDevice):
   """Represents a physical device and it's configuration."""
 
-  make: str = None
+  manufacturer: str = None
   model: str = None
   test_modules: str = None

framework/python/src/core/testrun.py

@@ -25,7 +25,7 @@
 import json
 import signal
 import time
-from common import logger
+from common import logger, util
 
 # Locate parent directory
 current_dir = os.path.dirname(os.path.realpath(__file__))
@@ -46,7 +46,7 @@
 LOCAL_DEVICES_DIR = 'local/devices'
 RESOURCE_DEVICES_DIR = 'resources/devices'
 DEVICE_CONFIG = 'device_config.json'
-DEVICE_MAKE = 'make'
+DEVICE_MANUFACTURER = 'manufacturer'
 DEVICE_MODEL = 'model'
 DEVICE_MAC_ADDR = 'mac_addr'
 DEVICE_TEST_MODULES = 'test_modules'
@@ -76,7 +76,6 @@ def __init__(self,
     self._net_orc = net_orc.NetworkOrchestrator(
       config_file=config_file_abs,
       validate=validate,
-      async_monitor=not self._net_only,
       single_intf = self._single_intf)
 
     self._test_orc = test_orc.TestOrchestrator(self._net_orc)
@@ -85,17 +84,30 @@ def start(self):
 
     self._load_all_devices()
 
+    self._start_network()
+
     if self._net_only:
       LOGGER.info('Network only option configured, no tests will be run')
-      self._start_network()
+
+      self._net_orc.listener.register_callback(
+        self._device_discovered,
+        [NetworkEvent.DEVICE_DISCOVERED]
+      )
+
+      self._net_orc.start_listener()
+      LOGGER.info('Waiting for devices on the network...')
+
+      while True:
+        time.sleep(RUNTIME)
+
     else:
-      self._start_network()
       self._test_orc.start()
 
       self._net_orc.listener.register_callback(
           self._device_stable,
           [NetworkEvent.DEVICE_STABLE]
       )
+
       self._net_orc.listener.register_callback(
         self._device_discovered,
         [NetworkEvent.DEVICE_DISCOVERED]
@@ -106,13 +118,13 @@ def start(self):
 
       time.sleep(RUNTIME)
 
-      if not self._test_orc.test_in_progress():
-        LOGGER.info('Timed out whilst waiting for device')
+      if not (self._test_orc.test_in_progress() or self._net_orc.monitor_in_progress()):
+        LOGGER.info('Timed out whilst waiting for device or stopping due to test completion')
       else:
-        while self._test_orc.test_in_progress():
+        while self._test_orc.test_in_progress() or self._net_orc.monitor_in_progress():
           time.sleep(5)
 
-    self.stop()
+      self.stop()
 
   def stop(self, kill=False):
     self._stop_tests()
@@ -157,18 +169,19 @@ def _load_devices(self, device_dir):
     LOGGER.debug('Loading devices from ' + device_dir)
 
     os.makedirs(device_dir, exist_ok=True)
+    util.run_command(f'chown -R {util.get_host_user()} {device_dir}')
 
     for device_folder in os.listdir(device_dir):
       with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG),
                 encoding='utf-8') as device_config_file:
         device_config_json = json.load(device_config_file)
 
-        device_make = device_config_json.get(DEVICE_MAKE)
+        device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER)
         device_model = device_config_json.get(DEVICE_MODEL)
         mac_addr = device_config_json.get(DEVICE_MAC_ADDR)
         test_modules = device_config_json.get(DEVICE_TEST_MODULES)
 
-        device = Device(make=device_make,
+        device = Device(manufacturer=device_manufacturer,
                         model=device_model,
                         mac_addr=mac_addr,
                         test_modules=json.dumps(test_modules))
@@ -184,7 +197,7 @@ def _device_discovered(self, mac_addr):
     device = self.get_device(mac_addr)
     if device is not None:
       LOGGER.info(
-        f'Discovered {device.make} {device.model} on the network')
+        f'Discovered {device.manufacturer} {device.model} on the network')
     else:
       device = Device(mac_addr=mac_addr)
       self._devices.append(device)

framework/python/src/net_orc/network_orchestrator.py

@@ -21,8 +21,6 @@
 import shutil
 import subprocess
 import sys
-import time
-import threading
 import docker
 from docker.types import Mount
 from common import logger
@@ -41,7 +39,6 @@
 TEST_DIR = 'test'
 MONITOR_PCAP = 'monitor.pcap'
 NET_DIR = 'runtime/network'
-#NETWORK_MODULES_DIR = 'network/modules'
 NETWORK_MODULES_DIR = 'modules/network'
 NETWORK_MODULE_METADATA = 'conf/module_config.json'
 DEVICE_BRIDGE = 'tr-d'
@@ -56,21 +53,18 @@
 DEFAULT_RUNTIME = 1200
 DEFAULT_MONITOR_PERIOD = 300
 
-RUNTIME = 1500
-
-
 class NetworkOrchestrator:
   """Manage and controls a virtual testing network."""
 
   def __init__(self,
                config_file=CONFIG_FILE,
                validate=True,
-               async_monitor=False,
                single_intf=False):
 
     self._runtime = DEFAULT_RUNTIME
     self._startup_timeout = DEFAULT_STARTUP_TIMEOUT
     self._monitor_period = DEFAULT_MONITOR_PERIOD
+    self._monitor_in_progress = False
 
     self._int_intf = None
     self._dev_intf = None
@@ -80,7 +74,6 @@ def __init__(self,
     self._net_modules = []
     self._devices = []
     self.validate = validate
-    self.async_monitor = async_monitor
 
     self._path = os.path.dirname(
         os.path.dirname(
@@ -99,7 +92,7 @@ def start(self):
 
     LOGGER.debug('Starting network orchestrator')
 
-    self._host_user = self._get_host_user()
+    self._host_user = util.get_host_user()
 
     # Get all components ready
     self.load_network_modules()
@@ -109,14 +102,6 @@ def start(self):
 
     self.start_network()
 
-    if self.async_monitor:
-      # Run the monitor method asynchronously to keep this method non-blocking
-      self._monitor_thread = threading.Thread(target=self.monitor_network)
-      self._monitor_thread.daemon = True
-      self._monitor_thread.start()
-    else:
-      self.monitor_network()
-
   def start_network(self):
     """Start the virtual testing network."""
     LOGGER.info('Starting network')
@@ -130,7 +115,7 @@ def start_network(self):
       self.validator.start()
 
     # Get network ready (via Network orchestrator)
-    LOGGER.info('Network is ready.')
+    LOGGER.debug('Network is ready')
 
   def start_listener(self):
     self.listener.start_listener()
@@ -151,13 +136,6 @@ def stop_network(self, kill=False):
     self.stop_networking_services(kill=kill)
     self.restore_net()
 
-  def monitor_network(self):
-    # TODO: This time should be configurable (How long to hold before exiting,
-    # this could be infinite too)
-    time.sleep(RUNTIME)
-
-    self.stop()
-
   def load_config(self, config_file=None):
     if config_file is None:
       # If not defined, use relative pathing to local file
@@ -178,8 +156,11 @@ def load_config(self, config_file=None):
 
   def _device_discovered(self, mac_addr):
 
+    self._monitor_in_progress = True
+
     LOGGER.debug(
         f'Discovered device {mac_addr}. Waiting for device to obtain IP')
+
     device = self._get_device(mac_addr=mac_addr)
 
     device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR,
@@ -204,6 +185,9 @@ def _device_discovered(self, mac_addr):
 
     self._start_device_monitor(device)
 
+  def monitor_in_progress(self):
+    return self._monitor_in_progress
+
   def _device_has_ip(self, packet):
     device = self._get_device(mac_addr=packet.src)
     if device is None or device.ip_addr is None:
@@ -225,6 +209,8 @@ def _start_device_monitor(self, device):
     wrpcap(
         os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''),
                      'monitor.pcap'), packet_capture)
+
+    self._monitor_in_progress = False
     self.listener.call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr)
 
   def _get_device(self, mac_addr):
@@ -490,46 +476,6 @@ def _start_network_service(self, net_module):
     if network != 'host':
       self._attach_service_to_network(net_module)
 
-  def _get_host_user(self):
-    user = self._get_os_user()
-
-    # If primary method failed, try secondary
-    if user is None:
-      user = self._get_user()
-
-    LOGGER.debug("Network orchestrator host user: " + user)
-    return user
-
-  def _get_os_user(self):
-    user = None
-    try:
-      user = os.getlogin()
-    except OSError as e:
-      # Handle the OSError exception
-      LOGGER.error("An OS error occurred while retrieving the login name.")
-    except Exception as e:
-      # Catch any other unexpected exceptions
-      LOGGER.error("An exception occurred:", e)
-    return user
-
-  def _get_user(self):
-    user = None
-    try:
-      user = getpass.getuser()
-    except (KeyError, ImportError, ModuleNotFoundError, OSError) as e:
-      # Handle specific exceptions individually
-      if isinstance(e, KeyError):
-        LOGGER.error("USER environment variable not set or unavailable.")
-      elif isinstance(e, ImportError):
-        LOGGER.error("Unable to import the getpass module.")
-      elif isinstance(e, ModuleNotFoundError):
-        LOGGER.error("The getpass module was not found.")
-      elif isinstance(e, OSError):
-        LOGGER.error("An OS error occurred while retrieving the username.")
-      else:
-        LOGGER.error("An exception occurred:", e)
-    return user
-
   def _stop_service_module(self, net_module, kill=False):
     LOGGER.debug('Stopping Service container ' + net_module.container_name)
     try: