Framework restructure

.gitignore

@@ -4,4 +4,5 @@ venv/
 error
 pylint.out
 local/
-__pycache__/
+__pycache__/
+build/

cmd/install

@@ -6,8 +6,4 @@ source venv/bin/activate
 
 pip3 install -r framework/requirements.txt
 
-pip3 install -r net_orc/python/requirements.txt
-
-pip3 install -r test_orc/python/requirements.txt
-
 deactivate

cmd/start

@@ -18,7 +18,9 @@ rm -rf runtime
 source venv/bin/activate
 
 # TODO: Execute python code
-python -u framework/test_runner.py $@
+# Set the PYTHONPATH to include the "src" directory
+export PYTHONPATH="$PWD/framework/python/src"
+python -u framework/python/src/core/test_runner.py $@
 
 # TODO: Work in progress code for containerization of OVS module
 # asyncRun() {

framework/logger.py → framework/python/src/common/logger.py

@@ -1,63 +1,60 @@
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Manages stream and file loggers."""
-import json
-import logging
-import os
-
-LOGGERS = {}
-_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s'
-_DATE_FORMAT = '%b %02d %H:%M:%S'
-_DEFAULT_LOG_LEVEL = logging.INFO
-_LOG_LEVEL = logging.INFO
-_CONF_DIR = 'conf'
-_CONF_FILE_NAME = 'system.json'
-_LOG_DIR = 'runtime/testing/'
-
-# Set log level
-with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME),
-          encoding='utf-8') as system_conf_file:
-  system_conf_json = json.load(system_conf_file)
-log_level_str = system_conf_json['log_level']
-
-temp_log = logging.getLogger('temp')
-try:
-  temp_log.setLevel(logging.getLevelName(log_level_str))
-  _LOG_LEVEL = logging.getLevelName(log_level_str)
-except ValueError:
-  print('Invalid log level set in ' + _CONF_DIR + '/' + _CONF_FILE_NAME +
-        '. Using INFO as log level')
-  _LOG_LEVEL = _DEFAULT_LOG_LEVEL
-
-log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT)
-
-def add_file_handler(log, log_file):
-  handler = logging.FileHandler(_LOG_DIR + log_file + '.log')
-  handler.setFormatter(log_format)
-  log.addHandler(handler)
-
-def add_stream_handler(log):
-  handler = logging.StreamHandler()
-  handler.setFormatter(log_format)
-  log.addHandler(handler)
-
-def get_logger(name, log_file=None):
-  if name not in LOGGERS:
-    LOGGERS[name] = logging.getLogger(name)
-    LOGGERS[name].setLevel(_LOG_LEVEL)
-    add_stream_handler(LOGGERS[name])
-  if log_file is not None:
-    add_file_handler(LOGGERS[name], log_file)
-  return LOGGERS[name]
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Sets up the logger to be used for the test modules."""
+import json
+import logging
+import os
+
+LOGGERS = {}
+_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s'
+_DATE_FORMAT = '%b %02d %H:%M:%S'
+_DEFAULT_LEVEL = logging.INFO
+_CONF_DIR = 'conf'
+_CONF_FILE_NAME = 'system.json'
+
+# Set log level
+try:
+  with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME),
+            encoding='UTF-8') as config_json_file:
+    system_conf_json = json.load(config_json_file)
+
+  log_level_str = system_conf_json['log_level']
+  log_level = logging.getLevelName(log_level_str)
+except OSError:
+  # TODO: Print out warning that log level is incorrect or missing
+  log_level = _DEFAULT_LEVEL
+
+log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT)
+
+def add_file_handler(log, log_file, log_dir):
+  handler = logging.FileHandler(log_dir + log_file + '.log')
+  handler.setFormatter(log_format)
+  log.addHandler(handler)
+
+
+def add_stream_handler(log):
+  handler = logging.StreamHandler()
+  handler.setFormatter(log_format)
+  log.addHandler(handler)
+
+
+def get_logger(name, log_file=None, log_dir=None):
+  if name not in LOGGERS:
+    LOGGERS[name] = logging.getLogger(name)
+    LOGGERS[name].setLevel(log_level)
+    add_stream_handler(LOGGERS[name])
+  if log_file is not None and log_dir is not None:
+    add_file_handler(LOGGERS[name], log_file, log_dir)
+  return LOGGERS[name]

net_orc/python/src/util.py → framework/python/src/common/util.py

@@ -1,55 +1,55 @@
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Provides basic utilities for the network orchestrator."""
-import subprocess
-import shlex
-import logger
-import netifaces
-
-LOGGER = logger.get_logger('util')
-
-
-def run_command(cmd, output=True):
-  """Runs a process at the os level
-  By default, returns the standard output and error output
-  If the caller sets optional output parameter to False,
-  will only return a boolean result indicating if it was
-  succesful in running the command.  Failure is indicated
-  by any return code from the process other than zero."""
-
-  success = False
-  process = subprocess.Popen(shlex.split(cmd),
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-
-  if process.returncode != 0 and output:
-    err_msg = f'{stderr.strip()}. Code: {process.returncode}'
-    LOGGER.error('Command Failed: ' + cmd)
-    LOGGER.error('Error: ' + err_msg)
-  else:
-    success = True
-  if output:
-    return stdout.strip().decode('utf-8'), stderr
-  else:
-    return success
-
-
-def interface_exists(interface):
-  return interface in netifaces.interfaces()
-
-
-def prettify(mac_string):
-  return ':'.join([f'{ord(b):02x}' for b in mac_string])
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides basic utilities for the network orchestrator."""
+import subprocess
+import shlex
+from common import logger
+import netifaces
+
+LOGGER = logger.get_logger('util')
+
+
+def run_command(cmd, output=True):
+  """Runs a process at the os level
+  By default, returns the standard output and error output
+  If the caller sets optional output parameter to False,
+  will only return a boolean result indicating if it was
+  succesful in running the command.  Failure is indicated
+  by any return code from the process other than zero."""
+
+  success = False
+  process = subprocess.Popen(shlex.split(cmd),
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+  stdout, stderr = process.communicate()
+
+  if process.returncode != 0 and output:
+    err_msg = f'{stderr.strip()}. Code: {process.returncode}'
+    LOGGER.error('Command Failed: ' + cmd)
+    LOGGER.error('Error: ' + err_msg)
+  else:
+    success = True
+  if output:
+    return stdout.strip().decode('utf-8'), stderr
+  else:
+    return success
+
+
+def interface_exists(interface):
+  return interface in netifaces.interfaces()
+
+
+def prettify(mac_string):
+  return ':'.join([f'{ord(b):02x}' for b in mac_string])

framework/device.py → framework/python/src/core/device.py

@@ -14,7 +14,7 @@
 
 """Track device object information."""
 
-from network_device import NetworkDevice
+from net_orc.network_device import NetworkDevice
 from dataclasses import dataclass
 
 

framework/test_runner.py → framework/python/src/core/test_runner.py

@@ -23,7 +23,7 @@
 import argparse
 import sys
 from testrun import TestRun
-import logger
+from common import logger
 import signal
 
 LOGGER = logger.get_logger("runner")

framework/testrun.py → framework/python/src/core/testrun.py

@@ -25,25 +25,18 @@
 import json
 import signal
 import time
-import logger
+from common import logger
 
 # Locate parent directory
 current_dir = os.path.dirname(os.path.realpath(__file__))
-parent_dir = os.path.dirname(current_dir)
 
-# Add net_orc to Python path
-net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src')
-sys.path.append(net_orc_dir)
+# Locate the test-run root directory, 4 levels, src->python->framework->test-run
+root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(current_dir))))
 
-# Add test_orc to Python path
-test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src')
-sys.path.append(test_orc_dir)
-
-from listener import NetworkEvent  # pylint: disable=wrong-import-position,import-outside-toplevel
-import test_orchestrator as test_orc  # pylint: disable=wrong-import-position,import-outside-toplevel
-import network_orchestrator as net_orc  # pylint: disable=wrong-import-position,import-outside-toplevel
-
-from device import Device # pylint: disable=wrong-import-position,import-outside-toplevel
+from net_orc.listener import NetworkEvent
+from test_orc import test_orchestrator as test_orc
+from net_orc import network_orchestrator as net_orc
+from device import Device
 
 LOGGER = logger.get_logger('test_run')
 CONFIG_FILE = 'conf/system.json'
@@ -58,7 +51,6 @@
 DEVICE_MAC_ADDR = 'mac_addr'
 DEVICE_TEST_MODULES = 'test_modules'
 
-
 class TestRun:  # pylint: disable=too-few-public-methods
   """Test Run controller.
 
@@ -142,7 +134,7 @@ def _exit_handler(self, signum, arg):  # pylint: disable=unused-argument
   def _get_config_abs(self, config_file=None):
     if config_file is None:
       # If not defined, use relative pathing to local file
-      config_file = os.path.join(parent_dir, CONFIG_FILE)
+      config_file = os.path.join(root_dir, CONFIG_FILE)
 
     # Expand the config file to absolute pathing
     return os.path.abspath(config_file)

net_orc/python/src/listener.py → framework/python/src/net_orc/listener.py

@@ -16,8 +16,8 @@
 under test."""
 import threading
 from scapy.all import AsyncSniffer, DHCP, get_if_hwaddr
-import logger
-from network_event import NetworkEvent
+from net_orc.network_event import NetworkEvent
+from common import logger
 
 LOGGER = logger.get_logger('listener')