From fc348c1e313b700babf9c0e240503b0fae3ef7dc Mon Sep 17 00:00:00 2001
From: jhughesbiot <jonathan.hughes@buildingsiot.com>
Date: Fri, 16 Jun 2023 11:05:09 -0600
Subject: [PATCH 1/6] Add initial work for ip control module

---
 framework/python/src/net_orc/ip_control.py    | 117 ++++++++++++++++++
 .../src/net_orc/network_orchestrator.py       |  66 ++++++----
 2 files changed, 157 insertions(+), 26 deletions(-)
 create mode 100644 framework/python/src/net_orc/ip_control.py

diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py
new file mode 100644
index 000000000..153d3c620
--- /dev/null
+++ b/framework/python/src/net_orc/ip_control.py
@@ -0,0 +1,117 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""IP Control Module"""
+from common import logger
+from common import util
+import re
+
+LOGGER = logger.get_logger('ip_ctrl')
+
+
+class IPControl:
+  """IP Control"""
+
+  def __init__(self):
+    """Initialize the IPControl object"""
+
+  def add_link(self, interface_name, peer_name):
+    """Create an ip link with a peer"""
+    success = util.run_command('ip link add ' + interface_name +
+                               ' type veth peer name ' + peer_name)
+    return success
+
+  def add_namespace(self, namespace):
+    """Add a network namespace"""
+    exists = self.namespace_exists(namespace)
+    if exists:
+      return True
+    else:
+      success = util.run_command('ip netns add ' + namespace)
+      return success
+
+  def namespace_exists(self,namespace):
+    """Check if a namespace already exists"""
+    namespaces = self.get_namespaces()
+    if namespace in namespaces:
+      return True
+    else:
+      return False
+
+  def get_namespaces(self):
+    stdout,stderr = util.run_command('ip netns list')
+    #Strip ID's from the namespace results
+    namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', stdout)
+    return namespaces
+
+  def set_namespace(self, interface_name, namespace):
+    """Attach an interface to a network namespace"""
+    success = util.run_command('ip link set ' + interface_name + ' netns ' +
+                               namespace)
+    return success
+
+  def rename_interface(self, interface_name, namespace, new_name):
+    """Rename an interface"""
+    success = util.run_command('ip netns exec ' + namespace +
+                               ' ip link set dev ' + interface_name + ' name ' +
+                               new_name)
+    return success
+
+  def set_interface_mac(self, interface_name, namespace, mac_addr):
+    """Set MAC address of an interface"""
+    success = util.run_command('ip netns exec ' + namespace +
+                               ' ip link set dev ' + interface_name +
+                               ' address ' + mac_addr)
+    return success
+
+  def set_interface_ip(self, interface_name, namespace, ipaddr):
+    """Set IP address of an interface"""
+    success = util.run_command('ip netns exec ' + namespace + ' ip addr add ' +
+                               ipaddr + ' dev ' + interface_name)
+    return success
+
+  def set_interface_up(self, interface_name, namespace=None):
+    """Set the interface to the up state"""
+    if namespace is None:
+      success = util.run_command('ip link set dev ' + interface_name + ' up')
+    else:
+      success = util.run_command('ip netns exec ' + namespace +
+                                 ' ip link set dev ' + interface_name + ' up')
+    return success
+
+  def configure_container_interface(self,bridge_intf, container_intf,
+                                    namespace_intf, namespace, mac_addr,
+                                    ipv4_addr, ipv6_addr):
+
+    # Create the interface pair
+    #self.add_link(bridge_intf,container_inf)
+
+    # Add the network namespace
+    self.add_namespace(namespace)
+
+    # Attach container interface to container network namespace
+    self.set_namespace(container_intf, namespace)
+
+    # Rename container interface name
+    self.rename_interface(container_intf,namespace, namespace_intf)
+
+    # Set MAC address of container interface
+    self.set_interface_mac(namespace_intf, namespace, mac_addr)
+
+    # Set IP address of container interface
+    self.set_interface_ip(namespace_intf, namespace, ipv4_addr)
+    self.set_interface_ip(namespace_intf, namespace, ipv6_addr)
+
+    # Set interfaces up
+    self.set_interface_up(bridge_intf)
+    self.set_interface_up(namespace_intf, namespace)
diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py
index f1f479742..77d9c013e 100644
--- a/framework/python/src/net_orc/network_orchestrator.py
+++ b/framework/python/src/net_orc/network_orchestrator.py
@@ -34,6 +34,7 @@
 from net_orc.network_event import NetworkEvent
 from net_orc.network_validator import NetworkValidator
 from net_orc.ovs_control import OVSControl
+from net_orc.ip_control import IPControl
 
 LOGGER = logger.get_logger('net_orc')
 CONFIG_FILE = 'conf/system.json'
@@ -92,6 +93,7 @@ def __init__(self,
     self.network_config = NetworkConfig()
     self.load_config(config_file)
     self._ovs = OVSControl()
+    self._ip_ctrl = IPControl()
 
   def start(self):
     """Start the network orchestrator."""
@@ -686,32 +688,44 @@ def _attach_service_to_network(self, net_module):
     util.run_command('ln -sf /proc/' + container_pid +
                      '/ns/net /var/run/netns/' + container_net_ns)
 
-    # Attach container interface to container network namespace
-    util.run_command('ip link set ' + container_intf + ' netns ' +
-                     container_net_ns)
-
-    # Rename container interface name to veth0
-    util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' +
-                     container_intf + ' name veth0')
-
-    # Set MAC address of container interface
-    util.run_command('ip netns exec ' + container_net_ns +
-                     ' ip link set dev veth0 address 9a:02:57:1e:8f:' +
-                     str(net_module.net_config.ip_index))
-
-    # Set IP address of container interface
-    util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' +
-                     net_module.net_config.get_ipv4_addr_with_prefix() +
-                     ' dev veth0')
-
-    util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' +
-                     net_module.net_config.get_ipv6_addr_with_prefix() +
-                     ' dev veth0')
-
-    # Set interfaces up
-    util.run_command('ip link set dev ' + bridge_intf + ' up')
-    util.run_command('ip netns exec ' + container_net_ns +
-                     ' ip link set dev veth0 up')
+    mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index)
+    ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix()
+    ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix()
+    self._ip_ctrl.configure_container_interface(
+      bridge_intf,
+      container_intf,
+      container_net_ns,
+      "veth0",
+      mac_addr,
+      ipv4_addr,
+      ipv6_addr)
+
+    # # Attach container interface to container network namespace
+    # util.run_command('ip link set ' + container_intf + ' netns ' +
+    #                  container_net_ns)
+
+    # # Rename container interface name to veth0
+    # util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' +
+    #                  container_intf + ' name veth0')
+
+    # # Set MAC address of container interface
+    # util.run_command('ip netns exec ' + container_net_ns +
+    #                  ' ip link set dev veth0 address 9a:02:57:1e:8f:' +
+    #                  str(net_module.net_config.ip_index))
+
+    # # Set IP address of container interface
+    # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' +
+    #                  net_module.net_config.get_ipv4_addr_with_prefix() +
+    #                  ' dev veth0')
+
+    # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' +
+    #                  net_module.net_config.get_ipv6_addr_with_prefix() +
+    #                  ' dev veth0')
+
+    # #Set interfaces up
+    # util.run_command('ip link set dev ' + bridge_intf + ' up')
+    # util.run_command('ip netns exec ' + container_net_ns +
+    #                  ' ip link set dev veth0 up')
 
     if net_module.net_config.enable_wan:
       LOGGER.debug('Attaching net service ' + net_module.display_name +

From a2bbe01fc5cd3fbe6cd3ddabf220a01cc2affbaf Mon Sep 17 00:00:00 2001
From: jhughesbiot <jonathan.hughes@buildingsiot.com>
Date: Fri, 16 Jun 2023 16:31:32 -0600
Subject: [PATCH 2/6] Implement ip control module with additional cleanup
 methods

---
 framework/python/src/net_orc/ip_control.py    |  70 ++++++++---
 .../src/net_orc/network_orchestrator.py       | 110 ++++++------------
 2 files changed, 90 insertions(+), 90 deletions(-)

diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py
index 153d3c620..93aba0351 100644
--- a/framework/python/src/net_orc/ip_control.py
+++ b/framework/python/src/net_orc/ip_control.py
@@ -34,12 +34,29 @@ def add_link(self, interface_name, peer_name):
   def add_namespace(self, namespace):
     """Add a network namespace"""
     exists = self.namespace_exists(namespace)
+    LOGGER.info("Namespace exists: " + str(exists))
     if exists:
       return True
     else:
       success = util.run_command('ip netns add ' + namespace)
       return success
 
+  def delete_link(self, interface_name):
+    """Delete an ip link"""
+    success = util.run_command('ip link delete ' + interface_name)
+    return success
+
+  def delete_namespace(self, interface_name):
+    """Delete an ip namespace"""
+    success = util.run_command('ip netns delete ' + interface_name)
+    return success  
+
+  def link_exists(self,interface_name):
+    stdout,stderr = util.run_command('ip link show ' + interface_name)
+    if "does not exist" in str(stderr):
+      return False
+    return True
+
   def namespace_exists(self,namespace):
     """Check if a namespace already exists"""
     namespaces = self.get_namespaces()
@@ -89,29 +106,54 @@ def set_interface_up(self, interface_name, namespace=None):
                                  ' ip link set dev ' + interface_name + ' up')
     return success
 
-  def configure_container_interface(self,bridge_intf, container_intf,
-                                    namespace_intf, namespace, mac_addr,
-                                    ipv4_addr, ipv6_addr):
+  def cleanup(self,interface=None,namespace=None):
+    """Cleanup existing link and namespace if they still exist"""
+    
+    link_clean = True
+    if interface is not None:
+      if self.link_exists(interface):
+        link_clean = self.delete_link(interface)
 
-    # Create the interface pair
-    #self.add_link(bridge_intf,container_inf)
+    ns_clean = True
+    if namespace is not None:
+      if self.namespace_exists(namespace):
+        ns_clean = self.delete_namespace
+    return link_clean and ns_clean
 
-    # Add the network namespace
-    self.add_namespace(namespace)
 
+  def configure_container_interface(self,bridge_intf, container_intf,
+                                    namespace_intf, namespace, mac_addr,
+                                    ipv4_addr=None, ipv6_addr=None):
     # Attach container interface to container network namespace
-    self.set_namespace(container_intf, namespace)
+    if not self.set_namespace(container_intf, namespace):
+      LOGGER.error(f'Failed to set namespace {namespace} for {container_intf}')
+      return False
 
     # Rename container interface name
-    self.rename_interface(container_intf,namespace, namespace_intf)
+    if not self.rename_interface(container_intf,namespace, namespace_intf):
+      LOGGER.error(f'Failed to rename container interface {container_intf} to {namespace_intf}')
+      return False
 
     # Set MAC address of container interface
-    self.set_interface_mac(namespace_intf, namespace, mac_addr)
+    if not self.set_interface_mac(namespace_intf, namespace, mac_addr):
+      LOGGER.error(f'Failed to set MAC address for {namespace_intf} to {mac_addr}')
+      return False
 
     # Set IP address of container interface
-    self.set_interface_ip(namespace_intf, namespace, ipv4_addr)
-    self.set_interface_ip(namespace_intf, namespace, ipv6_addr)
+    if ipv4_addr is not None:
+      if not self.set_interface_ip(namespace_intf, namespace, ipv4_addr):
+        LOGGER.error(f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}')
+        return False
+    if ipv6_addr is not None:
+      if not self.set_interface_ip(namespace_intf, namespace, ipv6_addr):
+        LOGGER.error(f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}')
+        return False
 
     # Set interfaces up
-    self.set_interface_up(bridge_intf)
-    self.set_interface_up(namespace_intf, namespace)
+    if not self.set_interface_up(bridge_intf):
+      LOGGER.error(f'Failed to set interface up {bridge_intf}')
+      return False
+    if not self.set_interface_up(namespace_intf, namespace):
+      LOGGER.error(f'Failed to set interface up {namespace_intf}')
+      return False
+    return True
diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py
index 77d9c013e..b6de1cd63 100644
--- a/framework/python/src/net_orc/network_orchestrator.py
+++ b/framework/python/src/net_orc/network_orchestrator.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Network orchestrator is responsible for managing
 all of the virtual network services"""
 import getpass
@@ -84,9 +83,10 @@ def __init__(self,
     self.validate = validate
     self.async_monitor = async_monitor
 
-    self._path = os.path.dirname(os.path.dirname(
-          os.path.dirname(
-            os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))
+    self._path = os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(
+                os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))
 
     self.validator = NetworkValidator()
     shutil.rmtree(os.path.join(os.getcwd(), NET_DIR), ignore_errors=True)
@@ -183,9 +183,8 @@ def _device_discovered(self, mac_addr):
         f'Discovered device {mac_addr}. Waiting for device to obtain IP')
     device = self._get_device(mac_addr=mac_addr)
 
-    device_runtime_dir = os.path.join(RUNTIME_DIR,
-        TEST_DIR,
-        device.mac_addr.replace(':', ''))
+    device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR,
+                                      device.mac_addr.replace(':', ''))
     os.makedirs(device_runtime_dir)
     util.run_command(f'chown -R {self._host_user} {device_runtime_dir}')
 
@@ -203,7 +202,7 @@ def _device_discovered(self, mac_addr):
     LOGGER.info(
         f'Device with mac addr {device.mac_addr} has obtained IP address '
         f'{device.ip_addr}')
-    
+
     self._start_device_monitor(device)
 
   def _device_has_ip(self, packet):
@@ -420,8 +419,7 @@ def _load_network_module(self, module_dir):
 
     # Determine if this is a template
     if 'template' in net_module_json['config']['docker']:
-      net_module.template = net_module_json['config']['docker'][
-          'template']
+      net_module.template = net_module_json['config']['docker']['template']
 
     # Load network service networking configuration
     if net_module.enable_container:
@@ -495,7 +493,7 @@ def _start_network_service(self, net_module):
 
   def _get_host_user(self):
     user = self._get_os_user()
-    
+
     # If primary method failed, try secondary
     if user is None:
       user = self._get_user()
@@ -512,7 +510,7 @@ def _get_os_user(self):
       LOGGER.error("An OS error occurred while retrieving the login name.")
     except Exception as e:
       # Catch any other unexpected exceptions
-       LOGGER.error("An exception occurred:", e)
+      LOGGER.error("An exception occurred:", e)
     return user
 
   def _get_user(self):
@@ -522,15 +520,15 @@ def _get_user(self):
     except (KeyError, ImportError, ModuleNotFoundError, OSError) as e:
       # Handle specific exceptions individually
       if isinstance(e, KeyError):
-          LOGGER.error("USER environment variable not set or unavailable.")
+        LOGGER.error("USER environment variable not set or unavailable.")
       elif isinstance(e, ImportError):
-          LOGGER.error("Unable to import the getpass module.")
+        LOGGER.error("Unable to import the getpass module.")
       elif isinstance(e, ModuleNotFoundError):
-          LOGGER.error("The getpass module was not found.")
+        LOGGER.error("The getpass module was not found.")
       elif isinstance(e, OSError):
-          LOGGER.error("An OS error occurred while retrieving the username.")
+        LOGGER.error("An OS error occurred while retrieving the username.")
       else:
-          LOGGER.error("An exception occurred:", e)
+        LOGGER.error("An exception occurred:", e)
     return user
 
   def _stop_service_module(self, net_module, kill=False):
@@ -668,9 +666,16 @@ def _attach_service_to_network(self, net_module):
     # Container network namespace name
     container_net_ns = 'tr-ctns-' + net_module.dir_name
 
+    # Resolve the interface information
+    mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index)
+    ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix()
+    ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix()
+
+    # Cleanup old interface and namespaces
+    self._ip_ctrl.cleanup(bridge_intf, container_net_ns)
+
     # Create interface pair
-    util.run_command('ip link add ' + bridge_intf + ' type veth peer name ' +
-                     container_intf)
+    self._ip_ctrl.add_link(bridge_intf, container_intf)
 
     # Add bridge interface to device bridge
     if self._ovs.add_port(port=bridge_intf, bridge_name=DEVICE_BRIDGE):
@@ -688,44 +693,9 @@ def _attach_service_to_network(self, net_module):
     util.run_command('ln -sf /proc/' + container_pid +
                      '/ns/net /var/run/netns/' + container_net_ns)
 
-    mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index)
-    ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix()
-    ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix()
-    self._ip_ctrl.configure_container_interface(
-      bridge_intf,
-      container_intf,
-      container_net_ns,
-      "veth0",
-      mac_addr,
-      ipv4_addr,
-      ipv6_addr)
-
-    # # Attach container interface to container network namespace
-    # util.run_command('ip link set ' + container_intf + ' netns ' +
-    #                  container_net_ns)
-
-    # # Rename container interface name to veth0
-    # util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' +
-    #                  container_intf + ' name veth0')
-
-    # # Set MAC address of container interface
-    # util.run_command('ip netns exec ' + container_net_ns +
-    #                  ' ip link set dev veth0 address 9a:02:57:1e:8f:' +
-    #                  str(net_module.net_config.ip_index))
-
-    # # Set IP address of container interface
-    # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' +
-    #                  net_module.net_config.get_ipv4_addr_with_prefix() +
-    #                  ' dev veth0')
-
-    # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' +
-    #                  net_module.net_config.get_ipv6_addr_with_prefix() +
-    #                  ' dev veth0')
-
-    # #Set interfaces up
-    # util.run_command('ip link set dev ' + bridge_intf + ' up')
-    # util.run_command('ip netns exec ' + container_net_ns +
-    #                  ' ip link set dev veth0 up')
+    self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
+                                                "veth0", container_net_ns,
+                                                mac_addr, ipv4_addr, ipv6_addr)
 
     if net_module.net_config.enable_wan:
       LOGGER.debug('Attaching net service ' + net_module.display_name +
@@ -739,9 +709,11 @@ def _attach_service_to_network(self, net_module):
       # tr-cti-dhcp (Test Run Container Interface for DHCP container)
       container_intf = 'tr-cti-' + net_module.dir_name
 
+      # Cleanup old interface
+      self._ip_ctrl.cleanup(bridge_intf)
+
       # Create interface pair
-      util.run_command('ip link add ' + bridge_intf + ' type veth peer name ' +
-                       container_intf)
+      self._ip_ctrl.add_link(bridge_intf, container_intf)
 
       # Attach bridge interface to internet bridge
       if self._ovs.add_port(port=bridge_intf, bridge_name=INTERNET_BRIDGE):
@@ -751,23 +723,9 @@ def _attach_service_to_network(self, net_module):
                        ' to internet bridge ' + DEVICE_BRIDGE + '. Exiting.')
           sys.exit(1)
 
-      # Attach container interface to container network namespace
-      util.run_command('ip link set ' + container_intf + ' netns ' +
-                       container_net_ns)
-
-      # Rename container interface name to eth1
-      util.run_command('ip netns exec ' + container_net_ns +
-                       ' ip link set dev ' + container_intf + ' name eth1')
-
-      # Set MAC address of container interface
-      util.run_command('ip netns exec ' + container_net_ns +
-                       ' ip link set dev eth1 address 9a:02:57:1e:8f:0' +
-                       str(net_module.net_config.ip_index))
-
-      # Set interfaces up
-      util.run_command('ip link set dev ' + bridge_intf + ' up')
-      util.run_command('ip netns exec ' + container_net_ns +
-                       ' ip link set dev eth1 up')
+      self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
+                                                "eth1", container_net_ns,
+                                                mac_addr)
 
   def restore_net(self):
 

From 016472b81c8c810bf1d87a7c8ee3128d56e706fa Mon Sep 17 00:00:00 2001
From: jhughesbiot <jonathan.hughes@buildingsiot.com>
Date: Mon, 19 Jun 2023 09:34:43 -0600
Subject: [PATCH 3/6] Update link check to not use error stream

---
 framework/python/src/net_orc/ip_control.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py
index 93aba0351..e0af93db1 100644
--- a/framework/python/src/net_orc/ip_control.py
+++ b/framework/python/src/net_orc/ip_control.py
@@ -51,11 +51,9 @@ def delete_namespace(self, interface_name):
     success = util.run_command('ip netns delete ' + interface_name)
     return success  
 
-  def link_exists(self,interface_name):
-    stdout,stderr = util.run_command('ip link show ' + interface_name)
-    if "does not exist" in str(stderr):
-      return False
-    return True
+  def link_exists(self,link_name):
+    links = self.get_links()
+    return link_name in links
 
   def namespace_exists(self,namespace):
     """Check if a namespace already exists"""
@@ -65,6 +63,20 @@ def namespace_exists(self,namespace):
     else:
       return False
 
+  def get_links(self):
+    stdout,stderr = util.run_command('ip link list')
+    links = stdout.strip().split('\n')
+    netns_links = []
+    for link in links:
+        match = re.search(r'\d+:\s+(\S+)', link)
+        if match:
+            interface_name = match.group(1)
+            name_match = re.search(r'(.*)@', interface_name)
+            if name_match:
+                interface_name = name_match.group(1)
+            netns_links.append(interface_name.strip())
+    return netns_links
+
   def get_namespaces(self):
     stdout,stderr = util.run_command('ip netns list')
     #Strip ID's from the namespace results

From 2d7b076f3346058d5f3d0a9cf10d9b74d101fd1e Mon Sep 17 00:00:00 2001
From: jhughesbiot <jonathan.hughes@buildingsiot.com>
Date: Mon, 19 Jun 2023 10:39:40 -0600
Subject: [PATCH 4/6] Add error checking around container network
 configurations

---
 framework/python/src/net_orc/ip_control.py    | 25 +++++++++++-
 .../src/net_orc/network_orchestrator.py       | 39 ++++++-------------
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py
index e0af93db1..46db690ea 100644
--- a/framework/python/src/net_orc/ip_control.py
+++ b/framework/python/src/net_orc/ip_control.py
@@ -135,7 +135,30 @@ def cleanup(self,interface=None,namespace=None):
 
   def configure_container_interface(self,bridge_intf, container_intf,
                                     namespace_intf, namespace, mac_addr,
-                                    ipv4_addr=None, ipv6_addr=None):
+                                    container_name=None,ipv4_addr=None, 
+                                    ipv6_addr=None):
+
+    # Cleanup old interface and namespaces
+    self.cleanup(bridge_intf, namespace)
+
+    # Create interface pair
+    self.add_link(bridge_intf, container_intf)
+
+    if container_name is not None:
+      # Get PID for running container
+      # TODO: Some error checking around missing PIDs might be required
+      container_pid = util.run_command('docker inspect -f {{.State.Pid}} ' +
+                                       container_name)[0]
+      if not container_pid.isdigit():
+        LOGGER.error(f'Failed to resolve pid for {container_name}')
+        return False
+
+      # Create symlink for container network namespace
+      if not util.run_command('ln -sf /proc/' + container_pid +
+                       '/ns/net /var/run/netns/' + namespace,output=False):
+        LOGGER.error(f'Failed to link {container_name} to namespace {namespace_intf}')
+        return False
+
     # Attach container interface to container network namespace
     if not self.set_namespace(container_intf, namespace):
       LOGGER.error(f'Failed to set namespace {namespace} for {container_intf}')
diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py
index b6de1cd63..475d2b9a2 100644
--- a/framework/python/src/net_orc/network_orchestrator.py
+++ b/framework/python/src/net_orc/network_orchestrator.py
@@ -671,11 +671,13 @@ def _attach_service_to_network(self, net_module):
     ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix()
     ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix()
 
-    # Cleanup old interface and namespaces
-    self._ip_ctrl.cleanup(bridge_intf, container_net_ns)
-
-    # Create interface pair
-    self._ip_ctrl.add_link(bridge_intf, container_intf)
+    # Add and configure the interface container
+    if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
+                                                "veth0", container_net_ns,
+                                                mac_addr,net_module.container_name,
+                                                ipv4_addr, ipv6_addr):
+      LOGGER.error('Failed to configure local networking for ' + net_module.name + '. Exiting.')
+      sys.exit(1)
 
     # Add bridge interface to device bridge
     if self._ovs.add_port(port=bridge_intf, bridge_name=DEVICE_BRIDGE):
@@ -684,19 +686,6 @@ def _attach_service_to_network(self, net_module):
                      DEVICE_BRIDGE + '. Exiting.')
         sys.exit(1)
 
-    # Get PID for running container
-    # TODO: Some error checking around missing PIDs might be required
-    container_pid = util.run_command('docker inspect -f {{.State.Pid}} ' +
-                                     net_module.container_name)[0]
-
-    # Create symlink for container network namespace
-    util.run_command('ln -sf /proc/' + container_pid +
-                     '/ns/net /var/run/netns/' + container_net_ns)
-
-    self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
-                                                "veth0", container_net_ns,
-                                                mac_addr, ipv4_addr, ipv6_addr)
-
     if net_module.net_config.enable_wan:
       LOGGER.debug('Attaching net service ' + net_module.display_name +
                    ' to internet bridge')
@@ -709,11 +698,11 @@ def _attach_service_to_network(self, net_module):
       # tr-cti-dhcp (Test Run Container Interface for DHCP container)
       container_intf = 'tr-cti-' + net_module.dir_name
 
-      # Cleanup old interface
-      self._ip_ctrl.cleanup(bridge_intf)
-
-      # Create interface pair
-      self._ip_ctrl.add_link(bridge_intf, container_intf)
+      if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
+                                                "eth1", container_net_ns,
+                                                mac_addr):
+        LOGGER.error('Failed to configure internet networking for ' + net_module.name + '. Exiting.')
+        sys.exit(1)
 
       # Attach bridge interface to internet bridge
       if self._ovs.add_port(port=bridge_intf, bridge_name=INTERNET_BRIDGE):
@@ -723,10 +712,6 @@ def _attach_service_to_network(self, net_module):
                        ' to internet bridge ' + DEVICE_BRIDGE + '. Exiting.')
           sys.exit(1)
 
-      self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
-                                                "eth1", container_net_ns,
-                                                mac_addr)
-
   def restore_net(self):
 
     LOGGER.info('Clearing baseline network')

From b71ba1da86462f0c81e5db6c29e0741118c0b7e7 Mon Sep 17 00:00:00 2001
From: jhughesbiot <jonathan.hughes@buildingsiot.com>
Date: Mon, 19 Jun 2023 11:15:21 -0600
Subject: [PATCH 5/6] Add network cleanup for namespaces and links

---
 framework/python/src/net_orc/ip_control.py     | 18 +++++++++++++++++-
 .../python/src/net_orc/network_orchestrator.py |  3 +++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py
index 46db690ea..eeac41475 100644
--- a/framework/python/src/net_orc/ip_control.py
+++ b/framework/python/src/net_orc/ip_control.py
@@ -118,6 +118,23 @@ def set_interface_up(self, interface_name, namespace=None):
                                  ' ip link set dev ' + interface_name + ' up')
     return success
 
+  def clean_all(self):
+    """Cleanup all existing test run interfaces and namespaces"""
+    
+    # Delete all namesapces that start with tr
+    namespaces = self.get_namespaces()
+    for ns in namespaces:
+      if 'tr' in ns:
+        self.delete_namespace(ns)
+
+    # Delete all namespaces that start with tr
+    links = self.get_links()
+    for link in links:
+      if 'tr' in link:
+        self.delete_link(link)
+
+
+
   def cleanup(self,interface=None,namespace=None):
     """Cleanup existing link and namespace if they still exist"""
     
@@ -132,7 +149,6 @@ def cleanup(self,interface=None,namespace=None):
         ns_clean = self.delete_namespace
     return link_clean and ns_clean
 
-
   def configure_container_interface(self,bridge_intf, container_intf,
                                     namespace_intf, namespace, mac_addr,
                                     container_name=None,ipv4_addr=None, 
diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py
index 475d2b9a2..6df2e334e 100644
--- a/framework/python/src/net_orc/network_orchestrator.py
+++ b/framework/python/src/net_orc/network_orchestrator.py
@@ -733,6 +733,9 @@ def restore_net(self):
     # Clear the virtual network
     self._ovs.restore_net()
 
+    # Clean up any existing network artifacts
+    self._ip_ctrl.clean_all()
+
     # Restart internet interface
     if util.interface_exists(self._int_intf):
       util.run_command('ip link set ' + self._int_intf + ' down')

From 386f41fa5a90edd097f0fe75b1763d3413419d67 Mon Sep 17 00:00:00 2001
From: jhughesbiot <jonathan.hughes@buildingsiot.com>
Date: Mon, 19 Jun 2023 11:29:30 -0600
Subject: [PATCH 6/6] formatting

---
 framework/python/src/net_orc/ip_control.py    | 64 +++++++++++--------
 .../src/net_orc/network_orchestrator.py       | 18 +++---
 2 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py
index eeac41475..eb683c46b 100644
--- a/framework/python/src/net_orc/ip_control.py
+++ b/framework/python/src/net_orc/ip_control.py
@@ -49,13 +49,13 @@ def delete_link(self, interface_name):
   def delete_namespace(self, interface_name):
     """Delete an ip namespace"""
     success = util.run_command('ip netns delete ' + interface_name)
-    return success  
+    return success
 
-  def link_exists(self,link_name):
+  def link_exists(self, link_name):
     links = self.get_links()
     return link_name in links
 
-  def namespace_exists(self,namespace):
+  def namespace_exists(self, namespace):
     """Check if a namespace already exists"""
     namespaces = self.get_namespaces()
     if namespace in namespaces:
@@ -64,21 +64,21 @@ def namespace_exists(self,namespace):
       return False
 
   def get_links(self):
-    stdout,stderr = util.run_command('ip link list')
+    stdout, stderr = util.run_command('ip link list')
     links = stdout.strip().split('\n')
     netns_links = []
     for link in links:
-        match = re.search(r'\d+:\s+(\S+)', link)
-        if match:
-            interface_name = match.group(1)
-            name_match = re.search(r'(.*)@', interface_name)
-            if name_match:
-                interface_name = name_match.group(1)
-            netns_links.append(interface_name.strip())
+      match = re.search(r'\d+:\s+(\S+)', link)
+      if match:
+        interface_name = match.group(1)
+        name_match = re.search(r'(.*)@', interface_name)
+        if name_match:
+          interface_name = name_match.group(1)
+        netns_links.append(interface_name.strip())
     return netns_links
 
   def get_namespaces(self):
-    stdout,stderr = util.run_command('ip netns list')
+    stdout, stderr = util.run_command('ip netns list')
     #Strip ID's from the namespace results
     namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', stdout)
     return namespaces
@@ -120,7 +120,7 @@ def set_interface_up(self, interface_name, namespace=None):
 
   def clean_all(self):
     """Cleanup all existing test run interfaces and namespaces"""
-    
+
     # Delete all namesapces that start with tr
     namespaces = self.get_namespaces()
     for ns in namespaces:
@@ -133,11 +133,9 @@ def clean_all(self):
       if 'tr' in link:
         self.delete_link(link)
 
-
-
-  def cleanup(self,interface=None,namespace=None):
+  def cleanup(self, interface=None, namespace=None):
     """Cleanup existing link and namespace if they still exist"""
-    
+
     link_clean = True
     if interface is not None:
       if self.link_exists(interface):
@@ -149,9 +147,14 @@ def cleanup(self,interface=None,namespace=None):
         ns_clean = self.delete_namespace
     return link_clean and ns_clean
 
-  def configure_container_interface(self,bridge_intf, container_intf,
-                                    namespace_intf, namespace, mac_addr,
-                                    container_name=None,ipv4_addr=None, 
+  def configure_container_interface(self,
+                                    bridge_intf,
+                                    container_intf,
+                                    namespace_intf,
+                                    namespace,
+                                    mac_addr,
+                                    container_name=None,
+                                    ipv4_addr=None,
                                     ipv6_addr=None):
 
     # Cleanup old interface and namespaces
@@ -171,8 +174,10 @@ def configure_container_interface(self,bridge_intf, container_intf,
 
       # Create symlink for container network namespace
       if not util.run_command('ln -sf /proc/' + container_pid +
-                       '/ns/net /var/run/netns/' + namespace,output=False):
-        LOGGER.error(f'Failed to link {container_name} to namespace {namespace_intf}')
+                              '/ns/net /var/run/netns/' + namespace,
+                              output=False):
+        LOGGER.error(
+            f'Failed to link {container_name} to namespace {namespace_intf}')
         return False
 
     # Attach container interface to container network namespace
@@ -181,23 +186,28 @@ def configure_container_interface(self,bridge_intf, container_intf,
       return False
 
     # Rename container interface name
-    if not self.rename_interface(container_intf,namespace, namespace_intf):
-      LOGGER.error(f'Failed to rename container interface {container_intf} to {namespace_intf}')
+    if not self.rename_interface(container_intf, namespace, namespace_intf):
+      LOGGER.error(
+          f'Failed to rename container interface {container_intf} to {namespace_intf}'
+      )
       return False
 
     # Set MAC address of container interface
     if not self.set_interface_mac(namespace_intf, namespace, mac_addr):
-      LOGGER.error(f'Failed to set MAC address for {namespace_intf} to {mac_addr}')
+      LOGGER.error(
+          f'Failed to set MAC address for {namespace_intf} to {mac_addr}')
       return False
 
     # Set IP address of container interface
     if ipv4_addr is not None:
       if not self.set_interface_ip(namespace_intf, namespace, ipv4_addr):
-        LOGGER.error(f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}')
+        LOGGER.error(
+            f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}')
         return False
     if ipv6_addr is not None:
       if not self.set_interface_ip(namespace_intf, namespace, ipv6_addr):
-        LOGGER.error(f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}')
+        LOGGER.error(
+            f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}')
         return False
 
     # Set interfaces up
diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py
index 6df2e334e..f3c07e8e4 100644
--- a/framework/python/src/net_orc/network_orchestrator.py
+++ b/framework/python/src/net_orc/network_orchestrator.py
@@ -672,11 +672,11 @@ def _attach_service_to_network(self, net_module):
     ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix()
 
     # Add and configure the interface container
-    if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
-                                                "veth0", container_net_ns,
-                                                mac_addr,net_module.container_name,
-                                                ipv4_addr, ipv6_addr):
-      LOGGER.error('Failed to configure local networking for ' + net_module.name + '. Exiting.')
+    if not self._ip_ctrl.configure_container_interface(
+        bridge_intf, container_intf, "veth0", container_net_ns, mac_addr,
+        net_module.container_name, ipv4_addr, ipv6_addr):
+      LOGGER.error('Failed to configure local networking for ' +
+                   net_module.name + '. Exiting.')
       sys.exit(1)
 
     # Add bridge interface to device bridge
@@ -698,10 +698,10 @@ def _attach_service_to_network(self, net_module):
       # tr-cti-dhcp (Test Run Container Interface for DHCP container)
       container_intf = 'tr-cti-' + net_module.dir_name
 
-      if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf,
-                                                "eth1", container_net_ns,
-                                                mac_addr):
-        LOGGER.error('Failed to configure internet networking for ' + net_module.name + '. Exiting.')
+      if not self._ip_ctrl.configure_container_interface(
+          bridge_intf, container_intf, "eth1", container_net_ns, mac_addr):
+        LOGGER.error('Failed to configure internet networking for ' +
+                     net_module.name + '. Exiting.')
         sys.exit(1)
 
       # Attach bridge interface to internet bridge