From fc348c1e313b700babf9c0e240503b0fae3ef7dc Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 16 Jun 2023 11:05:09 -0600 Subject: [PATCH 01/25] Add initial work for ip control module --- framework/python/src/net_orc/ip_control.py | 117 ++++++++++++++++++ .../src/net_orc/network_orchestrator.py | 66 ++++++---- 2 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 framework/python/src/net_orc/ip_control.py diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py new file mode 100644 index 000000000..153d3c620 --- /dev/null +++ b/framework/python/src/net_orc/ip_control.py @@ -0,0 +1,117 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""IP Control Module""" +from common import logger +from common import util +import re + +LOGGER = logger.get_logger('ip_ctrl') + + +class IPControl: + """IP Control""" + + def __init__(self): + """Initialize the IPControl object""" + + def add_link(self, interface_name, peer_name): + """Create an ip link with a peer""" + success = util.run_command('ip link add ' + interface_name + + ' type veth peer name ' + peer_name) + return success + + def add_namespace(self, namespace): + """Add a network namespace""" + exists = self.namespace_exists(namespace) + if exists: + return True + else: + success = util.run_command('ip netns add ' + namespace) + return success + + def namespace_exists(self,namespace): + """Check if a namespace already exists""" + namespaces = self.get_namespaces() + if namespace in namespaces: + return True + else: + return False + + def get_namespaces(self): + stdout,stderr = util.run_command('ip netns list') + #Strip ID's from the namespace results + namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', stdout) + return namespaces + + def set_namespace(self, interface_name, namespace): + """Attach an interface to a network namespace""" + success = util.run_command('ip link set ' + interface_name + ' netns ' + + namespace) + return success + + def rename_interface(self, interface_name, namespace, new_name): + """Rename an interface""" + success = util.run_command('ip netns exec ' + namespace + + ' ip link set dev ' + interface_name + ' name ' + + new_name) + return success + + def set_interface_mac(self, interface_name, namespace, mac_addr): + """Set MAC address of an interface""" + success = util.run_command('ip netns exec ' + namespace + + ' ip link set dev ' + interface_name + + ' address ' + mac_addr) + return success + + def set_interface_ip(self, interface_name, namespace, ipaddr): + """Set IP address of an interface""" + success = util.run_command('ip netns exec ' + namespace + ' ip addr add ' + + ipaddr + ' dev ' + interface_name) + return success + + def set_interface_up(self, interface_name, namespace=None): + """Set the interface to the up state""" + if namespace is None: + success = util.run_command('ip link set dev ' + interface_name + ' up') + else: + success = util.run_command('ip netns exec ' + namespace + + ' ip link set dev ' + interface_name + ' up') + return success + + def configure_container_interface(self,bridge_intf, container_intf, + namespace_intf, namespace, mac_addr, + ipv4_addr, ipv6_addr): + + # Create the interface pair + #self.add_link(bridge_intf,container_inf) + + # Add the network namespace + self.add_namespace(namespace) + + # Attach container interface to container network namespace + self.set_namespace(container_intf, namespace) + + # Rename container interface name + self.rename_interface(container_intf,namespace, namespace_intf) + + # Set MAC address of container interface + self.set_interface_mac(namespace_intf, namespace, mac_addr) + + # Set IP address of container interface + self.set_interface_ip(namespace_intf, namespace, ipv4_addr) + self.set_interface_ip(namespace_intf, namespace, ipv6_addr) + + # Set interfaces up + self.set_interface_up(bridge_intf) + self.set_interface_up(namespace_intf, namespace) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index f1f479742..77d9c013e 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -34,6 +34,7 @@ from net_orc.network_event import NetworkEvent from net_orc.network_validator import NetworkValidator from net_orc.ovs_control import OVSControl +from net_orc.ip_control import IPControl LOGGER = logger.get_logger('net_orc') CONFIG_FILE = 'conf/system.json' @@ -92,6 +93,7 @@ def __init__(self, self.network_config = NetworkConfig() self.load_config(config_file) self._ovs = OVSControl() + self._ip_ctrl = IPControl() def start(self): """Start the network orchestrator.""" @@ -686,32 +688,44 @@ def _attach_service_to_network(self, net_module): util.run_command('ln -sf /proc/' + container_pid + '/ns/net /var/run/netns/' + container_net_ns) - # Attach container interface to container network namespace - util.run_command('ip link set ' + container_intf + ' netns ' + - container_net_ns) - - # Rename container interface name to veth0 - util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' + - container_intf + ' name veth0') - - # Set MAC address of container interface - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev veth0 address 9a:02:57:1e:8f:' + - str(net_module.net_config.ip_index)) - - # Set IP address of container interface - util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + - net_module.net_config.get_ipv4_addr_with_prefix() + - ' dev veth0') - - util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + - net_module.net_config.get_ipv6_addr_with_prefix() + - ' dev veth0') - - # Set interfaces up - util.run_command('ip link set dev ' + bridge_intf + ' up') - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev veth0 up') + mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index) + ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix() + ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix() + self._ip_ctrl.configure_container_interface( + bridge_intf, + container_intf, + container_net_ns, + "veth0", + mac_addr, + ipv4_addr, + ipv6_addr) + + # # Attach container interface to container network namespace + # util.run_command('ip link set ' + container_intf + ' netns ' + + # container_net_ns) + + # # Rename container interface name to veth0 + # util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' + + # container_intf + ' name veth0') + + # # Set MAC address of container interface + # util.run_command('ip netns exec ' + container_net_ns + + # ' ip link set dev veth0 address 9a:02:57:1e:8f:' + + # str(net_module.net_config.ip_index)) + + # # Set IP address of container interface + # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + + # net_module.net_config.get_ipv4_addr_with_prefix() + + # ' dev veth0') + + # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + + # net_module.net_config.get_ipv6_addr_with_prefix() + + # ' dev veth0') + + # #Set interfaces up + # util.run_command('ip link set dev ' + bridge_intf + ' up') + # util.run_command('ip netns exec ' + container_net_ns + + # ' ip link set dev veth0 up') if net_module.net_config.enable_wan: LOGGER.debug('Attaching net service ' + net_module.display_name + From a2bbe01fc5cd3fbe6cd3ddabf220a01cc2affbaf Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 16 Jun 2023 16:31:32 -0600 Subject: [PATCH 02/25] Implement ip control module with additional cleanup methods --- framework/python/src/net_orc/ip_control.py | 70 ++++++++--- .../src/net_orc/network_orchestrator.py | 110 ++++++------------ 2 files changed, 90 insertions(+), 90 deletions(-) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index 153d3c620..93aba0351 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -34,12 +34,29 @@ def add_link(self, interface_name, peer_name): def add_namespace(self, namespace): """Add a network namespace""" exists = self.namespace_exists(namespace) + LOGGER.info("Namespace exists: " + str(exists)) if exists: return True else: success = util.run_command('ip netns add ' + namespace) return success + def delete_link(self, interface_name): + """Delete an ip link""" + success = util.run_command('ip link delete ' + interface_name) + return success + + def delete_namespace(self, interface_name): + """Delete an ip namespace""" + success = util.run_command('ip netns delete ' + interface_name) + return success + + def link_exists(self,interface_name): + stdout,stderr = util.run_command('ip link show ' + interface_name) + if "does not exist" in str(stderr): + return False + return True + def namespace_exists(self,namespace): """Check if a namespace already exists""" namespaces = self.get_namespaces() @@ -89,29 +106,54 @@ def set_interface_up(self, interface_name, namespace=None): ' ip link set dev ' + interface_name + ' up') return success - def configure_container_interface(self,bridge_intf, container_intf, - namespace_intf, namespace, mac_addr, - ipv4_addr, ipv6_addr): + def cleanup(self,interface=None,namespace=None): + """Cleanup existing link and namespace if they still exist""" + + link_clean = True + if interface is not None: + if self.link_exists(interface): + link_clean = self.delete_link(interface) - # Create the interface pair - #self.add_link(bridge_intf,container_inf) + ns_clean = True + if namespace is not None: + if self.namespace_exists(namespace): + ns_clean = self.delete_namespace + return link_clean and ns_clean - # Add the network namespace - self.add_namespace(namespace) + def configure_container_interface(self,bridge_intf, container_intf, + namespace_intf, namespace, mac_addr, + ipv4_addr=None, ipv6_addr=None): # Attach container interface to container network namespace - self.set_namespace(container_intf, namespace) + if not self.set_namespace(container_intf, namespace): + LOGGER.error(f'Failed to set namespace {namespace} for {container_intf}') + return False # Rename container interface name - self.rename_interface(container_intf,namespace, namespace_intf) + if not self.rename_interface(container_intf,namespace, namespace_intf): + LOGGER.error(f'Failed to rename container interface {container_intf} to {namespace_intf}') + return False # Set MAC address of container interface - self.set_interface_mac(namespace_intf, namespace, mac_addr) + if not self.set_interface_mac(namespace_intf, namespace, mac_addr): + LOGGER.error(f'Failed to set MAC address for {namespace_intf} to {mac_addr}') + return False # Set IP address of container interface - self.set_interface_ip(namespace_intf, namespace, ipv4_addr) - self.set_interface_ip(namespace_intf, namespace, ipv6_addr) + if ipv4_addr is not None: + if not self.set_interface_ip(namespace_intf, namespace, ipv4_addr): + LOGGER.error(f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}') + return False + if ipv6_addr is not None: + if not self.set_interface_ip(namespace_intf, namespace, ipv6_addr): + LOGGER.error(f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}') + return False # Set interfaces up - self.set_interface_up(bridge_intf) - self.set_interface_up(namespace_intf, namespace) + if not self.set_interface_up(bridge_intf): + LOGGER.error(f'Failed to set interface up {bridge_intf}') + return False + if not self.set_interface_up(namespace_intf, namespace): + LOGGER.error(f'Failed to set interface up {namespace_intf}') + return False + return True diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 77d9c013e..b6de1cd63 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Network orchestrator is responsible for managing all of the virtual network services""" import getpass @@ -84,9 +83,10 @@ def __init__(self, self.validate = validate self.async_monitor = async_monitor - self._path = os.path.dirname(os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) + self._path = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) self.validator = NetworkValidator() shutil.rmtree(os.path.join(os.getcwd(), NET_DIR), ignore_errors=True) @@ -183,9 +183,8 @@ def _device_discovered(self, mac_addr): f'Discovered device {mac_addr}. Waiting for device to obtain IP') device = self._get_device(mac_addr=mac_addr) - device_runtime_dir = os.path.join(RUNTIME_DIR, - TEST_DIR, - device.mac_addr.replace(':', '')) + device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, + device.mac_addr.replace(':', '')) os.makedirs(device_runtime_dir) util.run_command(f'chown -R {self._host_user} {device_runtime_dir}') @@ -203,7 +202,7 @@ def _device_discovered(self, mac_addr): LOGGER.info( f'Device with mac addr {device.mac_addr} has obtained IP address ' f'{device.ip_addr}') - + self._start_device_monitor(device) def _device_has_ip(self, packet): @@ -420,8 +419,7 @@ def _load_network_module(self, module_dir): # Determine if this is a template if 'template' in net_module_json['config']['docker']: - net_module.template = net_module_json['config']['docker'][ - 'template'] + net_module.template = net_module_json['config']['docker']['template'] # Load network service networking configuration if net_module.enable_container: @@ -495,7 +493,7 @@ def _start_network_service(self, net_module): def _get_host_user(self): user = self._get_os_user() - + # If primary method failed, try secondary if user is None: user = self._get_user() @@ -512,7 +510,7 @@ def _get_os_user(self): LOGGER.error("An OS error occurred while retrieving the login name.") except Exception as e: # Catch any other unexpected exceptions - LOGGER.error("An exception occurred:", e) + LOGGER.error("An exception occurred:", e) return user def _get_user(self): @@ -522,15 +520,15 @@ def _get_user(self): except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: # Handle specific exceptions individually if isinstance(e, KeyError): - LOGGER.error("USER environment variable not set or unavailable.") + LOGGER.error("USER environment variable not set or unavailable.") elif isinstance(e, ImportError): - LOGGER.error("Unable to import the getpass module.") + LOGGER.error("Unable to import the getpass module.") elif isinstance(e, ModuleNotFoundError): - LOGGER.error("The getpass module was not found.") + LOGGER.error("The getpass module was not found.") elif isinstance(e, OSError): - LOGGER.error("An OS error occurred while retrieving the username.") + LOGGER.error("An OS error occurred while retrieving the username.") else: - LOGGER.error("An exception occurred:", e) + LOGGER.error("An exception occurred:", e) return user def _stop_service_module(self, net_module, kill=False): @@ -668,9 +666,16 @@ def _attach_service_to_network(self, net_module): # Container network namespace name container_net_ns = 'tr-ctns-' + net_module.dir_name + # Resolve the interface information + mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index) + ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix() + ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix() + + # Cleanup old interface and namespaces + self._ip_ctrl.cleanup(bridge_intf, container_net_ns) + # Create interface pair - util.run_command('ip link add ' + bridge_intf + ' type veth peer name ' + - container_intf) + self._ip_ctrl.add_link(bridge_intf, container_intf) # Add bridge interface to device bridge if self._ovs.add_port(port=bridge_intf, bridge_name=DEVICE_BRIDGE): @@ -688,44 +693,9 @@ def _attach_service_to_network(self, net_module): util.run_command('ln -sf /proc/' + container_pid + '/ns/net /var/run/netns/' + container_net_ns) - mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index) - ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix() - ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix() - self._ip_ctrl.configure_container_interface( - bridge_intf, - container_intf, - container_net_ns, - "veth0", - mac_addr, - ipv4_addr, - ipv6_addr) - - # # Attach container interface to container network namespace - # util.run_command('ip link set ' + container_intf + ' netns ' + - # container_net_ns) - - # # Rename container interface name to veth0 - # util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' + - # container_intf + ' name veth0') - - # # Set MAC address of container interface - # util.run_command('ip netns exec ' + container_net_ns + - # ' ip link set dev veth0 address 9a:02:57:1e:8f:' + - # str(net_module.net_config.ip_index)) - - # # Set IP address of container interface - # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + - # net_module.net_config.get_ipv4_addr_with_prefix() + - # ' dev veth0') - - # util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + - # net_module.net_config.get_ipv6_addr_with_prefix() + - # ' dev veth0') - - # #Set interfaces up - # util.run_command('ip link set dev ' + bridge_intf + ' up') - # util.run_command('ip netns exec ' + container_net_ns + - # ' ip link set dev veth0 up') + self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, + "veth0", container_net_ns, + mac_addr, ipv4_addr, ipv6_addr) if net_module.net_config.enable_wan: LOGGER.debug('Attaching net service ' + net_module.display_name + @@ -739,9 +709,11 @@ def _attach_service_to_network(self, net_module): # tr-cti-dhcp (Test Run Container Interface for DHCP container) container_intf = 'tr-cti-' + net_module.dir_name + # Cleanup old interface + self._ip_ctrl.cleanup(bridge_intf) + # Create interface pair - util.run_command('ip link add ' + bridge_intf + ' type veth peer name ' + - container_intf) + self._ip_ctrl.add_link(bridge_intf, container_intf) # Attach bridge interface to internet bridge if self._ovs.add_port(port=bridge_intf, bridge_name=INTERNET_BRIDGE): @@ -751,23 +723,9 @@ def _attach_service_to_network(self, net_module): ' to internet bridge ' + DEVICE_BRIDGE + '. Exiting.') sys.exit(1) - # Attach container interface to container network namespace - util.run_command('ip link set ' + container_intf + ' netns ' + - container_net_ns) - - # Rename container interface name to eth1 - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev ' + container_intf + ' name eth1') - - # Set MAC address of container interface - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev eth1 address 9a:02:57:1e:8f:0' + - str(net_module.net_config.ip_index)) - - # Set interfaces up - util.run_command('ip link set dev ' + bridge_intf + ' up') - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev eth1 up') + self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, + "eth1", container_net_ns, + mac_addr) def restore_net(self): From 016472b81c8c810bf1d87a7c8ee3128d56e706fa Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 19 Jun 2023 09:34:43 -0600 Subject: [PATCH 03/25] Update link check to not use error stream --- framework/python/src/net_orc/ip_control.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index 93aba0351..e0af93db1 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -51,11 +51,9 @@ def delete_namespace(self, interface_name): success = util.run_command('ip netns delete ' + interface_name) return success - def link_exists(self,interface_name): - stdout,stderr = util.run_command('ip link show ' + interface_name) - if "does not exist" in str(stderr): - return False - return True + def link_exists(self,link_name): + links = self.get_links() + return link_name in links def namespace_exists(self,namespace): """Check if a namespace already exists""" @@ -65,6 +63,20 @@ def namespace_exists(self,namespace): else: return False + def get_links(self): + stdout,stderr = util.run_command('ip link list') + links = stdout.strip().split('\n') + netns_links = [] + for link in links: + match = re.search(r'\d+:\s+(\S+)', link) + if match: + interface_name = match.group(1) + name_match = re.search(r'(.*)@', interface_name) + if name_match: + interface_name = name_match.group(1) + netns_links.append(interface_name.strip()) + return netns_links + def get_namespaces(self): stdout,stderr = util.run_command('ip netns list') #Strip ID's from the namespace results From 2d7b076f3346058d5f3d0a9cf10d9b74d101fd1e Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 19 Jun 2023 10:39:40 -0600 Subject: [PATCH 04/25] Add error checking around container network configurations --- framework/python/src/net_orc/ip_control.py | 25 +++++++++++- .../src/net_orc/network_orchestrator.py | 39 ++++++------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index e0af93db1..46db690ea 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -135,7 +135,30 @@ def cleanup(self,interface=None,namespace=None): def configure_container_interface(self,bridge_intf, container_intf, namespace_intf, namespace, mac_addr, - ipv4_addr=None, ipv6_addr=None): + container_name=None,ipv4_addr=None, + ipv6_addr=None): + + # Cleanup old interface and namespaces + self.cleanup(bridge_intf, namespace) + + # Create interface pair + self.add_link(bridge_intf, container_intf) + + if container_name is not None: + # Get PID for running container + # TODO: Some error checking around missing PIDs might be required + container_pid = util.run_command('docker inspect -f {{.State.Pid}} ' + + container_name)[0] + if not container_pid.isdigit(): + LOGGER.error(f'Failed to resolve pid for {container_name}') + return False + + # Create symlink for container network namespace + if not util.run_command('ln -sf /proc/' + container_pid + + '/ns/net /var/run/netns/' + namespace,output=False): + LOGGER.error(f'Failed to link {container_name} to namespace {namespace_intf}') + return False + # Attach container interface to container network namespace if not self.set_namespace(container_intf, namespace): LOGGER.error(f'Failed to set namespace {namespace} for {container_intf}') diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index b6de1cd63..475d2b9a2 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -671,11 +671,13 @@ def _attach_service_to_network(self, net_module): ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix() ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix() - # Cleanup old interface and namespaces - self._ip_ctrl.cleanup(bridge_intf, container_net_ns) - - # Create interface pair - self._ip_ctrl.add_link(bridge_intf, container_intf) + # Add and configure the interface container + if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, + "veth0", container_net_ns, + mac_addr,net_module.container_name, + ipv4_addr, ipv6_addr): + LOGGER.error('Failed to configure local networking for ' + net_module.name + '. Exiting.') + sys.exit(1) # Add bridge interface to device bridge if self._ovs.add_port(port=bridge_intf, bridge_name=DEVICE_BRIDGE): @@ -684,19 +686,6 @@ def _attach_service_to_network(self, net_module): DEVICE_BRIDGE + '. Exiting.') sys.exit(1) - # Get PID for running container - # TODO: Some error checking around missing PIDs might be required - container_pid = util.run_command('docker inspect -f {{.State.Pid}} ' + - net_module.container_name)[0] - - # Create symlink for container network namespace - util.run_command('ln -sf /proc/' + container_pid + - '/ns/net /var/run/netns/' + container_net_ns) - - self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, - "veth0", container_net_ns, - mac_addr, ipv4_addr, ipv6_addr) - if net_module.net_config.enable_wan: LOGGER.debug('Attaching net service ' + net_module.display_name + ' to internet bridge') @@ -709,11 +698,11 @@ def _attach_service_to_network(self, net_module): # tr-cti-dhcp (Test Run Container Interface for DHCP container) container_intf = 'tr-cti-' + net_module.dir_name - # Cleanup old interface - self._ip_ctrl.cleanup(bridge_intf) - - # Create interface pair - self._ip_ctrl.add_link(bridge_intf, container_intf) + if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, + "eth1", container_net_ns, + mac_addr): + LOGGER.error('Failed to configure internet networking for ' + net_module.name + '. Exiting.') + sys.exit(1) # Attach bridge interface to internet bridge if self._ovs.add_port(port=bridge_intf, bridge_name=INTERNET_BRIDGE): @@ -723,10 +712,6 @@ def _attach_service_to_network(self, net_module): ' to internet bridge ' + DEVICE_BRIDGE + '. Exiting.') sys.exit(1) - self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, - "eth1", container_net_ns, - mac_addr) - def restore_net(self): LOGGER.info('Clearing baseline network') From b71ba1da86462f0c81e5db6c29e0741118c0b7e7 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 19 Jun 2023 11:15:21 -0600 Subject: [PATCH 05/25] Add network cleanup for namespaces and links --- framework/python/src/net_orc/ip_control.py | 18 +++++++++++++++++- .../python/src/net_orc/network_orchestrator.py | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index 46db690ea..eeac41475 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -118,6 +118,23 @@ def set_interface_up(self, interface_name, namespace=None): ' ip link set dev ' + interface_name + ' up') return success + def clean_all(self): + """Cleanup all existing test run interfaces and namespaces""" + + # Delete all namesapces that start with tr + namespaces = self.get_namespaces() + for ns in namespaces: + if 'tr' in ns: + self.delete_namespace(ns) + + # Delete all namespaces that start with tr + links = self.get_links() + for link in links: + if 'tr' in link: + self.delete_link(link) + + + def cleanup(self,interface=None,namespace=None): """Cleanup existing link and namespace if they still exist""" @@ -132,7 +149,6 @@ def cleanup(self,interface=None,namespace=None): ns_clean = self.delete_namespace return link_clean and ns_clean - def configure_container_interface(self,bridge_intf, container_intf, namespace_intf, namespace, mac_addr, container_name=None,ipv4_addr=None, diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 475d2b9a2..6df2e334e 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -733,6 +733,9 @@ def restore_net(self): # Clear the virtual network self._ovs.restore_net() + # Clean up any existing network artifacts + self._ip_ctrl.clean_all() + # Restart internet interface if util.interface_exists(self._int_intf): util.run_command('ip link set ' + self._int_intf + ' down') From 386f41fa5a90edd097f0fe75b1763d3413419d67 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 19 Jun 2023 11:29:30 -0600 Subject: [PATCH 06/25] formatting --- framework/python/src/net_orc/ip_control.py | 64 +++++++++++-------- .../src/net_orc/network_orchestrator.py | 18 +++--- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index eeac41475..eb683c46b 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -49,13 +49,13 @@ def delete_link(self, interface_name): def delete_namespace(self, interface_name): """Delete an ip namespace""" success = util.run_command('ip netns delete ' + interface_name) - return success + return success - def link_exists(self,link_name): + def link_exists(self, link_name): links = self.get_links() return link_name in links - def namespace_exists(self,namespace): + def namespace_exists(self, namespace): """Check if a namespace already exists""" namespaces = self.get_namespaces() if namespace in namespaces: @@ -64,21 +64,21 @@ def namespace_exists(self,namespace): return False def get_links(self): - stdout,stderr = util.run_command('ip link list') + stdout, stderr = util.run_command('ip link list') links = stdout.strip().split('\n') netns_links = [] for link in links: - match = re.search(r'\d+:\s+(\S+)', link) - if match: - interface_name = match.group(1) - name_match = re.search(r'(.*)@', interface_name) - if name_match: - interface_name = name_match.group(1) - netns_links.append(interface_name.strip()) + match = re.search(r'\d+:\s+(\S+)', link) + if match: + interface_name = match.group(1) + name_match = re.search(r'(.*)@', interface_name) + if name_match: + interface_name = name_match.group(1) + netns_links.append(interface_name.strip()) return netns_links def get_namespaces(self): - stdout,stderr = util.run_command('ip netns list') + stdout, stderr = util.run_command('ip netns list') #Strip ID's from the namespace results namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', stdout) return namespaces @@ -120,7 +120,7 @@ def set_interface_up(self, interface_name, namespace=None): def clean_all(self): """Cleanup all existing test run interfaces and namespaces""" - + # Delete all namesapces that start with tr namespaces = self.get_namespaces() for ns in namespaces: @@ -133,11 +133,9 @@ def clean_all(self): if 'tr' in link: self.delete_link(link) - - - def cleanup(self,interface=None,namespace=None): + def cleanup(self, interface=None, namespace=None): """Cleanup existing link and namespace if they still exist""" - + link_clean = True if interface is not None: if self.link_exists(interface): @@ -149,9 +147,14 @@ def cleanup(self,interface=None,namespace=None): ns_clean = self.delete_namespace return link_clean and ns_clean - def configure_container_interface(self,bridge_intf, container_intf, - namespace_intf, namespace, mac_addr, - container_name=None,ipv4_addr=None, + def configure_container_interface(self, + bridge_intf, + container_intf, + namespace_intf, + namespace, + mac_addr, + container_name=None, + ipv4_addr=None, ipv6_addr=None): # Cleanup old interface and namespaces @@ -171,8 +174,10 @@ def configure_container_interface(self,bridge_intf, container_intf, # Create symlink for container network namespace if not util.run_command('ln -sf /proc/' + container_pid + - '/ns/net /var/run/netns/' + namespace,output=False): - LOGGER.error(f'Failed to link {container_name} to namespace {namespace_intf}') + '/ns/net /var/run/netns/' + namespace, + output=False): + LOGGER.error( + f'Failed to link {container_name} to namespace {namespace_intf}') return False # Attach container interface to container network namespace @@ -181,23 +186,28 @@ def configure_container_interface(self,bridge_intf, container_intf, return False # Rename container interface name - if not self.rename_interface(container_intf,namespace, namespace_intf): - LOGGER.error(f'Failed to rename container interface {container_intf} to {namespace_intf}') + if not self.rename_interface(container_intf, namespace, namespace_intf): + LOGGER.error( + f'Failed to rename container interface {container_intf} to {namespace_intf}' + ) return False # Set MAC address of container interface if not self.set_interface_mac(namespace_intf, namespace, mac_addr): - LOGGER.error(f'Failed to set MAC address for {namespace_intf} to {mac_addr}') + LOGGER.error( + f'Failed to set MAC address for {namespace_intf} to {mac_addr}') return False # Set IP address of container interface if ipv4_addr is not None: if not self.set_interface_ip(namespace_intf, namespace, ipv4_addr): - LOGGER.error(f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}') + LOGGER.error( + f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}') return False if ipv6_addr is not None: if not self.set_interface_ip(namespace_intf, namespace, ipv6_addr): - LOGGER.error(f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}') + LOGGER.error( + f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}') return False # Set interfaces up diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 6df2e334e..f3c07e8e4 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -672,11 +672,11 @@ def _attach_service_to_network(self, net_module): ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix() # Add and configure the interface container - if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, - "veth0", container_net_ns, - mac_addr,net_module.container_name, - ipv4_addr, ipv6_addr): - LOGGER.error('Failed to configure local networking for ' + net_module.name + '. Exiting.') + if not self._ip_ctrl.configure_container_interface( + bridge_intf, container_intf, "veth0", container_net_ns, mac_addr, + net_module.container_name, ipv4_addr, ipv6_addr): + LOGGER.error('Failed to configure local networking for ' + + net_module.name + '. Exiting.') sys.exit(1) # Add bridge interface to device bridge @@ -698,10 +698,10 @@ def _attach_service_to_network(self, net_module): # tr-cti-dhcp (Test Run Container Interface for DHCP container) container_intf = 'tr-cti-' + net_module.dir_name - if not self._ip_ctrl.configure_container_interface(bridge_intf, container_intf, - "eth1", container_net_ns, - mac_addr): - LOGGER.error('Failed to configure internet networking for ' + net_module.name + '. Exiting.') + if not self._ip_ctrl.configure_container_interface( + bridge_intf, container_intf, "eth1", container_net_ns, mac_addr): + LOGGER.error('Failed to configure internet networking for ' + + net_module.name + '. Exiting.') sys.exit(1) # Attach bridge interface to internet bridge From 31572e5001f745663c95bf41eac93e73c4b106ab Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 26 Jun 2023 10:08:11 -0600 Subject: [PATCH 07/25] initial work on adding grpc functions for dhcp tests --- modules/network/base/base.Dockerfile | 4 + modules/network/base/python/requirements.txt | 3 +- modules/network/base/python/src/logger.py | 2 +- .../network/dhcp-1/bin/start_network_service | 5 +- modules/network/dhcp-1/conf/dhcpd.conf | 5 +- modules/network/dhcp-1/dhcp-1.Dockerfile | 6 + .../dhcp-1/python/src/grpc/dhcp_config.py | 144 ++++++++++--- .../python/src/grpc/dhcp_config_test.py | 30 +++ .../dhcp-1/python/src/grpc/dhcp_lease.py | 74 +++++++ .../dhcp-1/python/src/grpc/dhcp_leases.py | 201 ++++++++++++++++++ .../dhcp-1/python/src/grpc/network_service.py | 76 +++++-- .../dhcp-1/python/src/grpc/proto/grpc.proto | 20 +- .../network/dhcp-2/bin/start_network_service | 2 +- 13 files changed, 520 insertions(+), 52 deletions(-) create mode 100644 modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py create mode 100644 modules/network/dhcp-1/python/src/grpc/dhcp_lease.py create mode 100644 modules/network/dhcp-1/python/src/grpc/dhcp_leases.py diff --git a/modules/network/base/base.Dockerfile b/modules/network/base/base.Dockerfile index d14713c59..3bccb9774 100644 --- a/modules/network/base/base.Dockerfile +++ b/modules/network/base/base.Dockerfile @@ -3,10 +3,14 @@ FROM ubuntu:jammy ARG MODULE_NAME=base ARG MODULE_DIR=modules/network/$MODULE_NAME +ARG COMMON_DIR=framework/python/src/common # Install common software RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix +# Install common python modules +COPY $COMMON_DIR/ /testrun/python/src/common + # Setup the base python requirements COPY $MODULE_DIR/python /testrun/python diff --git a/modules/network/base/python/requirements.txt b/modules/network/base/python/requirements.txt index 9c4e2b056..9d9473d74 100644 --- a/modules/network/base/python/requirements.txt +++ b/modules/network/base/python/requirements.txt @@ -1,2 +1,3 @@ grpcio -grpcio-tools \ No newline at end of file +grpcio-tools +netifaces \ No newline at end of file diff --git a/modules/network/base/python/src/logger.py b/modules/network/base/python/src/logger.py index 8893b1e8d..998a4aaae 100644 --- a/modules/network/base/python/src/logger.py +++ b/modules/network/base/python/src/logger.py @@ -35,7 +35,7 @@ log_level = logging.getLevelName(log_level_str) except OSError: # TODO: Print out warning that log level is incorrect or missing - LOG_LEVEL = _DEFAULT_LEVEL + log_level = _DEFAULT_LEVEL log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) diff --git a/modules/network/dhcp-1/bin/start_network_service b/modules/network/dhcp-1/bin/start_network_service index a60806684..1d6a26d57 100644 --- a/modules/network/dhcp-1/bin/start_network_service +++ b/modules/network/dhcp-1/bin/start_network_service @@ -6,7 +6,7 @@ DHCP_LOG_FILE=/runtime/network/dhcp1-dhcpd.log RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp1-radvd.log -echo "Starrting Network Service..." +echo "Starting Network Service..." #Enable IPv6 Forwarding sysctl net.ipv6.conf.all.forwarding=1 @@ -15,6 +15,9 @@ sysctl -p # Create leases file if needed touch /var/lib/dhcp/dhcpd.leases +# Create log file if needed +touch /var/log/dhcp/dhcpd.log + #Create directory for radvd mkdir /var/run/radvd diff --git a/modules/network/dhcp-1/conf/dhcpd.conf b/modules/network/dhcp-1/conf/dhcpd.conf index 9f4fe1c28..cb631c91a 100644 --- a/modules/network/dhcp-1/conf/dhcpd.conf +++ b/modules/network/dhcp-1/conf/dhcpd.conf @@ -14,13 +14,16 @@ failover peer "failover-peer" { } subnet 10.10.10.0 netmask 255.255.255.0 { + authoritative; option ntp-servers 10.10.10.5; option subnet-mask 255.255.255.0; option broadcast-address 10.10.10.255; option routers 10.10.10.1; option domain-name-servers 10.10.10.4; + interface veth0; + pool { failover peer "failover-peer"; range 10.10.10.10 10.10.10.20; } -} +} \ No newline at end of file diff --git a/modules/network/dhcp-1/dhcp-1.Dockerfile b/modules/network/dhcp-1/dhcp-1.Dockerfile index 766f18c57..22430b90f 100644 --- a/modules/network/dhcp-1/dhcp-1.Dockerfile +++ b/modules/network/dhcp-1/dhcp-1.Dockerfile @@ -4,6 +4,12 @@ FROM test-run/base:latest ARG MODULE_NAME=dhcp-1 ARG MODULE_DIR=modules/network/$MODULE_NAME +# Install all necessary packages +RUN apt-get install -y wget + +#Update the oui.txt file from ieee +RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ + # Install dhcp server RUN apt-get install -y isc-dhcp-server radvd diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py index 99d6bdebd..b87a6e0df 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py @@ -27,43 +27,51 @@ class DHCPConfig: def __init__(self): self._default_lease_time = 300 - self.subnets = [] + self._subnets = [] self._peer = None + def disable_failover(self): + self._peer.disable() + for subnet in self._subnets: + subnet.disable_peer() + + def enable_failover(self): + self._peer.enable() + for subnet in self._subnets: + subnet.enable_peer() + def write_config(self): conf = str(self) - print('Writing config: \n' + conf) with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: conf_file.write(conf) - def resolve_config(self): - with open(CONFIG_FILE, 'r', encoding='UTF-8') as f: - conf = f.read() - self.resolve_subnets(conf) - self._peer = DHCPFailoverPeer(conf) + def resolve_config(self,config_file=CONFIG_FILE): + try: + with open(config_file, 'r', encoding='UTF-8') as f: + conf = f.read() + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + except Exception as e: + print("Failed to resolve config: " + str(e)) def resolve_subnets(self, conf): - self.subnets = [] + subnets = [] regex = r'(subnet.*)' - subnets = re.findall(regex, conf, re.MULTILINE | re.DOTALL) - for subnet in subnets: + subnets_conf = re.findall(regex, conf, re.MULTILINE | re.DOTALL) + for subnet in subnets_conf: dhcp_subnet = DHCPSubnet(subnet) - self.subnets.append(dhcp_subnet) + subnets.append(dhcp_subnet) + return subnets def set_range(self, start, end, subnet=0, pool=0): - print('Setting Range for pool ') - print(self.subnets[subnet].pools[pool]) - self.subnets[subnet].pools[pool].range_start = start - self.subnets[subnet].pools[pool].range_end = end + # Calculate the subnet from the range + octets = start.split('.') + octets[-1] = '0' + dhcp_subnet = '.'.join(octets) - # def resolve_settings(self, conf): - # lines = conf.split('\n') - # for line in lines: - # if DEFAULT_LEASE_TIME_KEY in line: - # self._default_lease_time = line.strip().split( - # DEFAULT_LEASE_TIME_KEY)[1].strip().split(';')[0] - - # self.peer = peer + #Update the subnet and range + self._subnets[subnet].set_subnet(dhcp_subnet) + self._subnets[subnet].pools[pool].set_range(start,end) def __str__(self): @@ -73,7 +81,7 @@ def __str__(self): DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, DEFAULT_LEASE_TIME=self._default_lease_time) - config += '\n\n' + str(self.peer) + config += '\n\n' + str(self._peer) for subnet in self._subnets: config += '\n\n' + str(subnet) return str(config) @@ -108,6 +116,7 @@ def __init__(self, config): self.split = None self.load_balance_max_seconds = None self.peer = None + self.enabled = True self.resolve_peer(config) @@ -123,9 +132,9 @@ def __str__(self): {MCLT_KEY} {MCLT}; {SPLIT_KEY} {SPLIT}; {LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS}; - \r}}""" + \r\n}}""" - return config.format( + config = config.format( length='multi-line', FAILOVER_PEER_KEY=FAILOVER_PEER_KEY, FAILOVER_PEER=self.name, @@ -148,6 +157,20 @@ def __str__(self): LOAD_BALANCE_MAX_SECONDS_KEY=LOAD_BALANCE_MAX_SECONDS_KEY, LOAD_BALANCE_MAX_SECONDS=self.load_balance_max_seconds) + if not self.enabled: + lines = config.strip().split('\n') + for i in range(len(lines)): + lines[i] = '#' + lines[i] + config = '\n'.join(lines) + + return config + + def disable(self): + self.enabled=False + + def enable(self): + self.enabled=True + def resolve_peer(self, conf): peer = '' lines = conf.split('\n') @@ -187,37 +210,45 @@ def resolve_peer(self, conf): break self.peer = peer - +SUBNET_KEY = 'subnet' NTP_OPTION_KEY = 'option ntp-servers' SUBNET_MASK_OPTION_KEY = 'option subnet-mask' BROADCAST_OPTION_KEY = 'option broadcast-address' ROUTER_OPTION_KEY = 'option routers' DNS_OPTION_KEY = 'option domain-name-servers' +INTERFACE_KEY = 'interface' +AUTHORITATIVE_KEY = 'authoritative' class DHCPSubnet: """Represents the DHCP Servers subnet configuration""" def __init__(self, subnet): + self._authoritative = False + self._subnet = None self._ntp_servers = None self._subnet_mask = None self._broadcast = None self._routers = None self._dns_servers = None + self._interface = None self.pools = [] self.resolve_subnet(subnet) self.resolve_pools(subnet) def __str__(self): - config = """subnet 10.10.10.0 netmask {SUBNET_MASK_OPTION} {{ + config = """subnet {SUBNET_OPTION} netmask {SUBNET_MASK_OPTION} {{ \r\t{NTP_OPTION_KEY} {NTP_OPTION}; \r\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION}; \r\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION}; \r\t{ROUTER_OPTION_KEY} {ROUTER_OPTION}; - \r\t{DNS_OPTION_KEY} {DNS_OPTION};""" + \r\t{DNS_OPTION_KEY} {DNS_OPTION}; + \r\t{INTERFACE_KEY} {INTERFACE_OPTION}; + \r\tauthoritative;""" config = config.format(length='multi-line', + SUBNET_OPTION =self._subnet, NTP_OPTION_KEY=NTP_OPTION_KEY, NTP_OPTION=self._ntp_servers, SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, @@ -227,17 +258,46 @@ def __str__(self): ROUTER_OPTION_KEY=ROUTER_OPTION_KEY, ROUTER_OPTION=self._routers, DNS_OPTION_KEY=DNS_OPTION_KEY, - DNS_OPTION=self._dns_servers) + DNS_OPTION=self._dns_servers, + INTERFACE_KEY=INTERFACE_KEY, + INTERFACE_OPTION=self._interface) + + if not self._authoritative: + config = config.replace(AUTHORITATIVE_KEY,'#'+AUTHORITATIVE_KEY) + for pool in self.pools: config += '\n\t' + str(pool) config += '\n\r}' return config + def disable_peer(self): + for pool in self.pools: + pool.disable_peer() + + def enable_peer(self): + for pool in self.pools: + pool.enable_peer() + + def set_subnet(self,subnet,netmask=None): + if netmask is None: + netmask = '255.255.255.0' + self._subnet = subnet + self._subnet_mask = netmask + + # Calculate the broadcast from the subnet + octets = subnet.split('.') + octets[-1] = '255' + dhcp_broadcast = '.'.join(octets) + + self._broadcast = dhcp_broadcast + def resolve_subnet(self, subnet): subnet_parts = subnet.split('\n') for part in subnet_parts: - if NTP_OPTION_KEY in part: + if part.strip().startswith(SUBNET_KEY): + self._subnet = part.strip().split()[1] + elif NTP_OPTION_KEY in part: self._ntp_servers = part.strip().split(NTP_OPTION_KEY)[1].strip().split( ';')[0] elif SUBNET_MASK_OPTION_KEY in part: @@ -252,6 +312,11 @@ def resolve_subnet(self, subnet): elif DNS_OPTION_KEY in part: self._dns_servers = part.strip().split(DNS_OPTION_KEY)[1].strip().split( ';')[0] + elif INTERFACE_KEY in part: + self._interface = part.strip().split(INTERFACE_KEY)[1].strip().split( + ';')[0] + elif AUTHORITATIVE_KEY in part: + self._authoritative = True def resolve_pools(self, subnet): regex = r'(pool.*)\}' @@ -273,6 +338,7 @@ def __init__(self, pool): self.range_start = None self.range_end = None self.resolve_pool(pool) + self._peer_enabled = True def __str__(self): @@ -281,7 +347,7 @@ def __str__(self): \r\t\t{RANGE_KEY} {RANGE_START} {RANGE_END}; \r\t}}""" - return config.format( + config = config.format( length='multi-line', FAILOVER_KEY=FAILOVER_KEY, FAILOVER=self.failover_peer, @@ -290,6 +356,20 @@ def __str__(self): RANGE_END=self.range_end, ) + if not self._peer_enabled: + config = config.replace(FAILOVER_KEY,'#'+FAILOVER_KEY) + return config + + def disable_peer(self): + self._peer_enabled = False + + def enable_peer(self): + self._peer_enabled = True + + def set_range(self,start,end): + self.range_start=start + self.range_end=end + def resolve_pool(self, pool): pool_parts = pool.split('\n') # pool_parts = pool.split("\n") diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py new file mode 100644 index 000000000..139c82e59 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py @@ -0,0 +1,30 @@ +import unittest +from dhcp_config import DHCPConfig +import os + +CONFIG_FILE_TEST = '../../../conf/dhcpd.conf' + +class DHCPConfigTest(unittest.TestCase): + + + def test_resolve_config(self): + dhcp_config = DHCPConfig() + path = os.path.abspath(CONFIG_FILE_TEST) + dhcp_config.resolve_config(CONFIG_FILE_TEST) + + dhcp_config.set_range('10.0.0.20','10.0.0.30') + #print('Modified Subnet:\n' + str(dhcp_config)) + + def test_disable_failover(self): + dhcp_config = DHCPConfig() + path = os.path.abspath(CONFIG_FILE_TEST) + dhcp_config.resolve_config(CONFIG_FILE_TEST) + dhcp_config.disable_failover() + print('Disabled Peer:\n' + str(dhcp_config)) + + dhcp_config.enable_failover() + print('Enabled Peer:\n' + str(dhcp_config)) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py b/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py new file mode 100644 index 000000000..aa47745c6 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py @@ -0,0 +1,74 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import time + +timeFormat = "%Y-%m-%d %H:%M:%S" + + +class DHCPLease(object): + hw_addr = None + ip = None + hostname = None + expires = None + + def __init__(self, lease): + self._make_lease(lease) + + def _make_lease(self, lease): + if lease is not None: + sectionsRaw = lease.split(" ") + sections = [] + for section in sectionsRaw: + if not (not section.strip()): + sections.append(section) + self.hw_addr = sections[0] + self.ip = sections[1] + self.hostname = sections[2] + self.expires = sections[3] + " " + sections[4] + self.manufacturer = " ".join(sections[5:]) + + + def get_millis(self, timestamp): + dt_obj = datetime.strptime(timestamp, timeFormat) + millis = dt_obj.timestamp() * 1000 + return millis + + def get_expires_millis(self): + return self.get_millis(self.expires) + + def isExpired(self): + expiresMilis = self.get_expires_millis() + curTime = int(round(time.time()) * 1000) + return curTime >= expiresMilis + + def __str__(self): + lease = {} + if self.hw_addr is not None: + lease['hw_addr'] = self.hw_addr + + if self.ip is not None: + lease['ip'] = self.ip + + if self.hostname is not None: + lease['hostname'] = self.hostname + + if self.expires is not None: + lease['expires'] = self.expires + + if self.manufacturer is not None: + lease['manufacturer'] = self.manufacturer + + return str(lease) \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py b/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py new file mode 100644 index 000000000..6ea9ce543 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py @@ -0,0 +1,201 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Used to resolve the DHCP servers lease information""" + +import os +import sys +from dhcp_lease import DHCPLease + +# Add the parent directory to sys.path +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.insert(0, parent_dir) + +import logger +from common import util + +LOG_NAME = "dhcp_lease" +LOGGER = None + +DHCP_LEASE_FILES = [ + "/var/lib/dhcp/dhcpd.leases", "/var/lib/dhcp/dhcpd.leases~", + "/var/lib/dhcp/dhcpd6.leases", "/var/lib/dhcp/dhcpd6.leases~" +] +DHCP_CONFIG_FILE = '/etc/dhcp/dhcpd.conf' + +reservedHostTemplate=""" +host HOSTNAME{ + hardware ethernet HW_ADDR; + fixed-address RESERVED_IP; +}""" + +class DHCPLeases: + """Leases for the DHCP server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def enable_failover(self): + config = self._get_config() + try: + LOGGER.info('Enabling failover peer config block') + failoverStart = config.find('failover peer') - 1 + failoverEnd = config.find('}',failoverStart) + failover = config[failoverStart:failoverEnd+1] + lines = failover.split('\n') + for i in range(len(lines)): + if lines[i].strip().startswith('#'): + lines[i] = lines[i].replace('#','',1) + enabled_config = '\n'.join(lines) + new_config = config[0:failoverStart] + enabled_config + config[failoverEnd+1:] + LOGGER.info('Failover peer config block enabled') + + LOGGER.info('Enabling all failover references') + lines = new_config.split('\n') + for i in range(len(lines)): + if lines[i].strip().startswith('#') and 'failover peer' in lines[i]: + lines[i] = lines[i].replace('#','',1) + LOGGER.info('Failover references enabled') + + new_config = '\n'.join(lines) + self._write_config(new_config) + + except Exception as e: + LOGGER.error("Could not disable failover : " + str(e)) + + def disable_failover(self): + config = self._get_config() + try: + LOGGER.info('Disabling failover peer config block') + failoverStart = config.find('failover peer') + failoverEnd = config.find('}',failoverStart) + failover = config[failoverStart:failoverEnd+1] + lines = failover.split('\n') + for i in range(len(lines)): + if not lines[i].strip().startswith('#'): + lines[i] = '#' + lines[i] + disabled_config = '\n'.join(lines) + new_config = config[0:failoverStart] + disabled_config + config[failoverEnd+1:] + LOGGER.info('Failover peer config block disabled') + + LOGGER.info('Disabling all failover references') + lines = new_config.split('\n') + for i in range(len(lines)): + if lines[i].strip().startswith('failover peer'): + lines[i] = '#' + lines[i] + LOGGER.info('Failover references disabled') + + new_config = '\n'.join(lines) + self._write_config(new_config) + + except Exception as e: + LOGGER.error("Could not disable failover : " + str(e)) + + def add_reserved_host(self,hostname,hw_addr,ip_addr): + self.delete_reserved_host(hw_addr) + LOGGER.info("Add Reserved Host: " + hostname + ":" + hw_addr + ":" + ip_addr) + reservedHost = reservedHostTemplate.replace("HOSTNAME",hostname) + reservedHost = reservedHost.replace("HW_ADDR",hw_addr) + reservedHost = reservedHost.replace("RESERVED_IP",ip_addr) + newConfig = self._get_config() + reservedHost + self._write_config(newConfig) + + def delete_reserved_host(self,hw_addr): + config = self._get_config() + try: + ixHw = config.find("hardware ethernet " + hw_addr) + hostStart = config.rindex("host",0,ixHw) + hostEnd = config.find("}",hostStart) + newConfig = config[0:hostStart] + config[hostEnd+1:] + self._write_config(newConfig) + except Exception as e: + LOGGER.error("Could not delete host: " + hw_addr + str(e)) + + def delete_all_hosts(self): + LOGGER.info("Deleting hosts") + for lease in DHCP_LEASE_FILES: + LOGGER.info("Checking file: " + lease) + if os.path.exists(lease): + LOGGER.info("File Exists: " + lease) + try: + # Delete existing lease file + os.remove(lease) + except OSError as e: + LOGGER.info(f"Error occurred while deleting the file: {e}") + # Create an empty lease file + with open(lease,'w'): + pass + + def _get_config(self): + content = None + with open(DHCP_CONFIG_FILE,"r") as f: + content= f.read() + return content + + def get_lease(self,hw_addr): + for lease in self.get_leases(): + if lease.hw_addr == hw_addr: + return lease + + def get_leases(self): + leases = [] + lease_list_raw = self._get_lease_list() + LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + "\n") + lines = lease_list_raw.split( + '===============================================================================================' + )[1].split('\n') + for line in lines: + try: + lease = DHCPLease(line) + leases.append(lease) + except Exception as e: + # Let non lease lines file without extra checks + LOGGER.error("Making Lease Error: " + str(e)) + LOGGER.error("Not a valid lease line: " + line) + return leases + + def delete_lease(self,ip_addr): + LOGGER.info("Deleting lease") + for lease in DHCP_LEASE_FILES: + LOGGER.info("Checking file: " + lease) + if os.path.exists(lease): + LOGGER.info("File Exists: " + lease) + try: + # Delete existing lease file + with (open(lease,'r')) as f: + contents = f.read() + + while ip_addr in contents: + ixIp = contents.find(ip_addr) + leaseStart = contents.rindex("lease",0,ixIp) + leaseEnd = contents.find("}",leaseStart) + LOGGER.info("Lease Location: " + str(leaseStart)+":"+str(leaseEnd)) + contents = contents[0:leaseStart] + contents[leaseEnd+1:] + + except OSError as e: + LOGGER.info(f"Error occurred while deleting the lease: {e}") + + + def _get_lease_list(self): + LOGGER.info('Running lease list command') + try: + stdout, stderr = util.run_command('dhcp-lease-list') + return stdout + except Exception as e: + LOGGER.error("Error lease list: " + str(e)) + + def _write_config(self,config): + with open(DHCP_CONFIG_FILE,"w") as f: + f.write(config) \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc/network_service.py index 64aab8a07..3ee5fb92a 100644 --- a/modules/network/dhcp-1/python/src/grpc/network_service.py +++ b/modules/network/dhcp-1/python/src/grpc/network_service.py @@ -11,44 +11,92 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """gRPC Network Service for the DHCP Server network module""" import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 from dhcp_config import DHCPConfig +from dhcp_leases import DHCPLeases class NetworkService(pb2_grpc.NetworkModule): """gRPC endpoints for the DHCP Server""" def __init__(self): - self._dhcp_config = DHCPConfig() + self._dhcp_config = None + self.dhcp_leases = DHCPLeases() + + def _get_dhcp_config(self): + if self._dhcp_config is None: + self._dhcp_config = DHCPConfig() + self._dhcp_config.resolve_config() + return self._dhcp_config + + def AddReservedLease(self, request, context): # pylint: disable=W0613 + print("Add Reserved Lease Called") + try: + self.dhcp_leases.add_reserved_host(request.hostname,request.hw_addr,request.ip_addr) + print("Reserve Leased Finished") + except Exception as e: + print("Failed: " + str(e)) + return pb2.Response(code=200, message='{}') + + def DisableFailover(self, request, contest): # pylint: disable=W0613 + print("Disabling Failover") + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.disable_failover() + dhcp_config.write_config() + print("Failover Disabled") + except Exception as e: + print("Failed: " + str(e)) + return pb2.Response(code=200, message='{}') + + def EnableFailover(self, request, contest): # pylint: disable=W0613 + print("Enable Failover") + try: + dhcp_config.enable_failover() + dhcp_config.write_config() + print("Failover Enabled") + except Exception as e: + print("Failed: " + str(e)) + return pb2.Response(code=200, message='{}') + + def GetIPAddress(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP leased address for the + provided MAC address + """ + lease = self.dhcp_leases.get_lease(request.hw_addr) + if lease is not None: + return pb2.Response(code=200, message=str(lease)) + else: + return pb2.Response(code=200, message='{}') - def GetDHCPRange(self, request, context): # pylint: disable=W0613 + def GetDHCPRange(self, request, context): # pylint: disable=W0613 """ Resolve the current DHCP configuration and return the first range from the first subnet in the file """ - self._dhcp_config.resolve_config() - pool = self._dhcp_config.subnets[0].pools[0] + pool = self.get_dhcp_config().subnets[0].pools[0] return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) - def SetDHCPRange(self, request, context): # pylint: disable=W0613 + def SetDHCPRange(self, request, context): # pylint: disable=W0613 """ Change DHCP configuration and set the the first range from the first subnet in the configuration """ - print('Setting DHCPRange') - print('Start: ' + request.start) - print('End: ' + request.end) - self._dhcp_config.resolve_config() - self._dhcp_config.set_range(request.start, request.end, 0, 0) - self._dhcp_config.write_config() - return pb2.Response(code=200, message='DHCP Range Set') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.set_range(request.start, request.end, 0, 0) + dhcp_config.write_config() + return pb2.Response(code=200, message='DHCP Range Set') + except Exception as e: + print("Failed: " + str(e)) + return pb2.Response(code=200, message='{}') - def GetStatus(self, request, context): # pylint: disable=W0613 + def GetStatus(self, request, context): # pylint: disable=W0613 """ Return the current status of the network module """ diff --git a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto index 8e2732620..79749d8a4 100644 --- a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto +++ b/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto @@ -12,6 +12,12 @@ service NetworkModule { rpc SetLeaseAddress(SetLeaseAddressRequest) returns (Response) {}; + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; + + rpc DisableFailover(DisableFailoverRequest) returns (Response) {}; + + rpc EnableFailover(EnableFailoverRequest) returns (Response) {}; + } message Response { @@ -25,12 +31,24 @@ message DHCPRange { string end = 3; } +message AddReservedLeaseRequest { + string hostname = 1; + string hw_addr = 2; + string ip_addr = 3; +} + message GetDHCPRangeRequest {} -message GetIPAddressRequest {} +message GetIPAddressRequest { + string hw_addr = 1; +} message GetStatusRequest {} +message DisableFailoverRequest {} + +message EnableFailoverRequest {} + message SetLeaseAddressRequest { string ipAddress = 1; } \ No newline at end of file diff --git a/modules/network/dhcp-2/bin/start_network_service b/modules/network/dhcp-2/bin/start_network_service index ad5ff09e7..59db31ddf 100644 --- a/modules/network/dhcp-2/bin/start_network_service +++ b/modules/network/dhcp-2/bin/start_network_service @@ -6,7 +6,7 @@ DHCP_LOG_FILE=/runtime/network/dhcp2-dhcpd.log RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp2-radvd.log -echo "Starrting Network Service..." +echo "Starting Network Service..." #Enable IPv6 Forwarding sysctl net.ipv6.conf.all.forwarding=1 From b965bb14d388fe6d2e78e23d13d79a138a92f2f2 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 28 Jun 2023 09:26:17 -0600 Subject: [PATCH 08/25] rework code to allow for better usage and unit testing --- modules/network/dhcp-1/conf/dhcpd.conf | 27 ++- .../dhcp-1/python/src/grpc/dhcp_config.py | 167 +++++++++++++++--- .../python/src/grpc/dhcp_config_test.py | 88 +++++++-- .../dhcp-1/python/src/grpc/network_service.py | 22 ++- run_tests.sh | 4 + 5 files changed, 256 insertions(+), 52 deletions(-) create mode 100644 run_tests.sh diff --git a/modules/network/dhcp-1/conf/dhcpd.conf b/modules/network/dhcp-1/conf/dhcpd.conf index cb631c91a..38eeaf693 100644 --- a/modules/network/dhcp-1/conf/dhcpd.conf +++ b/modules/network/dhcp-1/conf/dhcpd.conf @@ -1,29 +1,28 @@ default-lease-time 300; failover peer "failover-peer" { - primary; - address 10.10.10.2; - port 847; - peer address 10.10.10.3; - peer port 647; - max-response-delay 60; - max-unacked-updates 10; - mclt 3600; - split 128; - load balance max seconds 3; + primary; + address 10.10.10.2; + port 847; + peer address 10.10.10.3; + peer port 647; + max-response-delay 60; + max-unacked-updates 10; + mclt 3600; + split 128; + load balance max seconds 3; } subnet 10.10.10.0 netmask 255.255.255.0 { - authoritative; option ntp-servers 10.10.10.5; option subnet-mask 255.255.255.0; option broadcast-address 10.10.10.255; option routers 10.10.10.1; option domain-name-servers 10.10.10.4; interface veth0; - + authoritative; pool { - failover peer "failover-peer"; - range 10.10.10.10 10.10.10.20; + failover peer "failover-peer"; + range 10.10.10.10 10.10.10.20; } } \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py index b87a6e0df..af5350a06 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py @@ -15,12 +15,30 @@ """Contains all the necessary classes to maintain the DHCP server's configuration""" import re +import os +import sys + +# Add the parent directory to sys.path +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +sys.path.insert(0, parent_dir) + +import common.logger as logger + +LOG_NAME = "dhcp_config" +LOGGER = None CONFIG_FILE = '/etc/dhcp/dhcpd.conf' CONFIG_FILE_TEST = 'network/modules/dhcp-1/conf/dhcpd.conf' DEFAULT_LEASE_TIME_KEY = 'default-lease-time' +RESERVED_HOST_TEMPLATE = """ +host {HOSTNAME}{{ + hardware ethernet {HW_ADDR}; + fixed-address {RESERVED_IP}; +}}""" + class DHCPConfig: """Represents the DHCP Servers configuration and gives access to modify it""" @@ -29,6 +47,18 @@ def __init__(self): self._default_lease_time = 300 self._subnets = [] self._peer = None + self._reserved_hosts = [] + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def add_reserved_host(self, hostname, hw_addr, ip_addr): + host = DHCPReservedHost(host=hostname,hw_addr=hw_addr,fixed_addr=ip_addr) + self._reserved_hosts.append(host) + + def delete_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host._hw_addr: + self._reserved_hosts.remove(host) def disable_failover(self): self._peer.disable() @@ -40,17 +70,40 @@ def enable_failover(self): for subnet in self._subnets: subnet.enable_peer() - def write_config(self): - conf = str(self) - with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: - conf_file.write(conf) + def get_reserved_host(self,hw_addr): + for host in self._reserved_hosts: + if hw_addr == host._hw_addr: + return host + + def write_config(self,config=None): + if config is None: + conf = str(self) + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(conf) + else: + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(config) + + def _get_config(self, config_file=CONFIG_FILE): + content = None + with open(config_file, "r") as f: + content = f.read() + return content + + def make(self,conf): + try: + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: + print("Failed to make DHCPConfig: " + str(e)) - def resolve_config(self,config_file=CONFIG_FILE): + def resolve_config(self, config_file=CONFIG_FILE): try: - with open(config_file, 'r', encoding='UTF-8') as f: - conf = f.read() + conf = self._get_config(config_file) self._subnets = self.resolve_subnets(conf) self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) except Exception as e: print("Failed to resolve config: " + str(e)) @@ -63,6 +116,20 @@ def resolve_subnets(self, conf): subnets.append(dhcp_subnet) return subnets + def resolve_reserved_hosts(self,conf): + hosts = [] + host_start = 0 + while True: + host_start = conf.find('host',host_start) + if host_start < 0: + break + else: + host_end = conf.find('}',host_start) + host = DHCPReservedHost(config=conf[host_start:host_end+1]) + hosts.append(host) + host_start = host_end+1 + return hosts + def set_range(self, start, end, subnet=0, pool=0): # Calculate the subnet from the range octets = start.split('.') @@ -71,19 +138,27 @@ def set_range(self, start, end, subnet=0, pool=0): #Update the subnet and range self._subnets[subnet].set_subnet(dhcp_subnet) - self._subnets[subnet].pools[pool].set_range(start,end) + self._subnets[subnet].pools[pool].set_range(start, end) def __str__(self): - config = """\r{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" - + # Encode the top level config options + config = """{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" config = config.format(length='multi-line', DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, DEFAULT_LEASE_TIME=self._default_lease_time) + # Encode the failover peer config += '\n\n' + str(self._peer) + + # Encode the subnets for subnet in self._subnets: config += '\n\n' + str(subnet) + + # Encode the reserved hosts + for host in self._reserved_hosts: + config += '\n' + str(host) + return str(config) @@ -132,7 +207,7 @@ def __str__(self): {MCLT_KEY} {MCLT}; {SPLIT_KEY} {SPLIT}; {LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS}; - \r\n}}""" + \r}}""" config = config.format( length='multi-line', @@ -166,10 +241,10 @@ def __str__(self): return config def disable(self): - self.enabled=False + self.enabled = False def enable(self): - self.enabled=True + self.enabled = True def resolve_peer(self, conf): peer = '' @@ -210,6 +285,7 @@ def resolve_peer(self, conf): break self.peer = peer + SUBNET_KEY = 'subnet' NTP_OPTION_KEY = 'option ntp-servers' SUBNET_MASK_OPTION_KEY = 'option subnet-mask' @@ -248,7 +324,7 @@ def __str__(self): \r\tauthoritative;""" config = config.format(length='multi-line', - SUBNET_OPTION =self._subnet, + SUBNET_OPTION=self._subnet, NTP_OPTION_KEY=NTP_OPTION_KEY, NTP_OPTION=self._ntp_servers, SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, @@ -263,12 +339,12 @@ def __str__(self): INTERFACE_OPTION=self._interface) if not self._authoritative: - config = config.replace(AUTHORITATIVE_KEY,'#'+AUTHORITATIVE_KEY) + config = config.replace(AUTHORITATIVE_KEY, '#' + AUTHORITATIVE_KEY) for pool in self.pools: config += '\n\t' + str(pool) - config += '\n\r}' + config += '\n}' return config def disable_peer(self): @@ -279,7 +355,7 @@ def enable_peer(self): for pool in self.pools: pool.enable_peer() - def set_subnet(self,subnet,netmask=None): + def set_subnet(self, subnet, netmask=None): if netmask is None: netmask = '255.255.255.0' self._subnet = subnet @@ -357,7 +433,8 @@ def __str__(self): ) if not self._peer_enabled: - config = config.replace(FAILOVER_KEY,'#'+FAILOVER_KEY) + config = config.replace(FAILOVER_KEY, '#' + FAILOVER_KEY) + return config def disable_peer(self): @@ -366,13 +443,12 @@ def disable_peer(self): def enable_peer(self): self._peer_enabled = True - def set_range(self,start,end): - self.range_start=start - self.range_end=end + def set_range(self, start, end): + self.range_start = start + self.range_end = end def resolve_pool(self, pool): pool_parts = pool.split('\n') - # pool_parts = pool.split("\n") for part in pool_parts: if FAILOVER_KEY in part: self.failover_peer = part.strip().split(FAILOVER_KEY)[1].strip().split( @@ -381,3 +457,50 @@ def resolve_pool(self, pool): pool_range = part.strip().split(RANGE_KEY)[1].strip().split(';')[0] self.range_start = pool_range.split(' ')[0].strip() self.range_end = pool_range.split(' ')[1].strip() + + +HOST_KEY = 'host' +HARDWARE_KEY = 'hardware ethernet' +FIXED_ADDRESS_KEY = 'fixed-address' + +class DHCPReservedHost: + """Represents a DHCP Servers subnet pool configuration""" + + def __init__(self, host=None,hw_addr=None,fixed_addr=None,config=None): + if config is None: + self._host = host + self._hw_addr = hw_addr + self._fixed_addr = fixed_addr + else: + self.resolve_host(config) + + def __str__(self): + + config = """{HOST_KEY} {HOSTNAME} {{ + \r\t{HARDWARE_KEY} {HW_ADDR}; + \r\t{FIXED_ADDRESS_KEY} {RESERVED_IP}; + \r}}""" + + config = config.format( + length='multi-line', + HOST_KEY=HOST_KEY, + HOSTNAME=self._host, + HARDWARE_KEY=HARDWARE_KEY, + HW_ADDR=self._hw_addr, + FIXED_ADDRESS_KEY=FIXED_ADDRESS_KEY, + RESERVED_IP=self._fixed_addr, + ) + return config + + def resolve_host(self, reserved_host): + host_parts = reserved_host.split('\n') + for part in host_parts: + if HOST_KEY in part: + self._host = part.strip().split(HOST_KEY)[1].strip().split( + '{')[0] + elif HARDWARE_KEY in part: + self._hw_addr = part.strip().split(HARDWARE_KEY)[1].strip().split( + ';')[0] + elif FIXED_ADDRESS_KEY in part: + self._fixed_addr = part.strip().split(FIXED_ADDRESS_KEY)[1].strip().split( + ';')[0] diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py index 139c82e59..66b2c42e6 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py @@ -2,29 +2,89 @@ from dhcp_config import DHCPConfig import os -CONFIG_FILE_TEST = '../../../conf/dhcpd.conf' +CONFIG_FILE = 'conf/dhcpd.conf' + +DHCP_CONFIG = None + +def get_config_file_path(): + dhcp_config = DHCPConfig() + current_dir = os.path.dirname(os.path.abspath(__file__)) + module_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(current_dir)))) + conf_file = os.path.join(module_dir,CONFIG_FILE) + return conf_file + +def get_config(): + dhcp_config = DHCPConfig() + dhcp_config.resolve_config(get_config_file_path()) + return dhcp_config class DHCPConfigTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Resolve the config + global DHCP_CONFIG + DHCP_CONFIG = get_config() def test_resolve_config(self): - dhcp_config = DHCPConfig() - path = os.path.abspath(CONFIG_FILE_TEST) - dhcp_config.resolve_config(CONFIG_FILE_TEST) + print('Test Resolve Config:\n' + str(DHCP_CONFIG)) + + # Resolve the raw config file + with open(get_config_file_path(),'r') as f: + lines = f.readlines() - dhcp_config.set_range('10.0.0.20','10.0.0.30') - #print('Modified Subnet:\n' + str(dhcp_config)) + # Get the resolved config as a + conf_parts = str(DHCP_CONFIG).split('\n') + # dhcpd conf is not picky about spacing so we just + # need to check contents of each line for matching + # to make sure evertying matches + for i in range(len(lines)): + self.assertEqual(lines[i].strip(),conf_parts[i].strip()) + def test_disable_failover(self): - dhcp_config = DHCPConfig() - path = os.path.abspath(CONFIG_FILE_TEST) - dhcp_config.resolve_config(CONFIG_FILE_TEST) - dhcp_config.disable_failover() - print('Disabled Peer:\n' + str(dhcp_config)) + DHCP_CONFIG.disable_failover() + print('Test Disable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertTrue(line.startswith('#')) - dhcp_config.enable_failover() - print('Enabled Peer:\n' + str(dhcp_config)) + def test_enable_failover(self): + DHCP_CONFIG.enable_failover() + print('Test Enable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertFalse(line.startswith('#')) + def test_add_reserved_host(self): + DHCP_CONFIG.add_reserved_host('test','00:11:22:33:44:55','192.168.10.5') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print('AddHostConfig:\n' + str(DHCP_CONFIG)) + + def test_delete_reserved_host(self): + DHCP_CONFIG.delete_reserved_host('00:11:22:33:44:55') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNone(host) + print('DeleteHostConfig:\n' + str(DHCP_CONFIG)) + + def test_resolve_config_with_hosts(self): + DHCP_CONFIG.add_reserved_host('test','00:11:22:33:44:55','192.168.10.5') + config_with_hosts = DHCPConfig() + config_with_hosts.make(str(DHCP_CONFIG)) + host = config_with_hosts.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print("ResolveConfigWithHosts:\n" + str(config_with_hosts)) if __name__ == '__main__': - unittest.main() \ No newline at end of file + suite = unittest.TestSuite() + suite.addTest(DHCPConfigTest('test_resolve_config')) + suite.addTest(DHCPConfigTest('test_disable_failover')) + suite.addTest(DHCPConfigTest('test_enable_failover')) + suite.addTest(DHCPConfigTest('test_add_reserved_host')) + suite.addTest(DHCPConfigTest('test_delete_reserved_host')) + suite.addTest(DHCPConfigTest('test_resolve_config_with_hosts')) + + runner = unittest.TextTestRunner() + runner.run(suite) + # unittest.main() \ No newline at end of file diff --git a/modules/network/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc/network_service.py index 3ee5fb92a..38c0e2fae 100644 --- a/modules/network/dhcp-1/python/src/grpc/network_service.py +++ b/modules/network/dhcp-1/python/src/grpc/network_service.py @@ -18,6 +18,8 @@ from dhcp_config import DHCPConfig from dhcp_leases import DHCPLeases +import traceback + class NetworkService(pb2_grpc.NetworkModule): """gRPC endpoints for the DHCP Server""" @@ -35,10 +37,25 @@ def _get_dhcp_config(self): def AddReservedLease(self, request, context): # pylint: disable=W0613 print("Add Reserved Lease Called") try: - self.dhcp_leases.add_reserved_host(request.hostname,request.hw_addr,request.ip_addr) - print("Reserve Leased Finished") + dhcp_config = self._get_dhcp_config() + dhcp_config.add_reserved_host(request.hostname,request.hw_addr,request.ip_addr) + dhcp_config.write_config() + print("Reserve Leased Added") except Exception as e: print("Failed: " + str(e)) + traceback.print_exc() + return pb2.Response(code=200, message='{}') + + def DeleteReservedLease(self, request, context): # pylint: disable=W0613 + print("Delete Reserved Lease Called") + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.delete_reserved_host(request.hw_addr) + dhcp_config.write_config() + print("Reserve Leased Deleted") + except Exception as e: + print("Failed: " + str(e)) + traceback.print_exc() return pb2.Response(code=200, message='{}') def DisableFailover(self, request, contest): # pylint: disable=W0613 @@ -55,6 +72,7 @@ def DisableFailover(self, request, contest): # pylint: disable=W0613 def EnableFailover(self, request, contest): # pylint: disable=W0613 print("Enable Failover") try: + dhcp_config = self._get_dhcp_config() dhcp_config.enable_failover() dhcp_config.write_config() print("Failover Enabled") diff --git a/run_tests.sh b/run_tests.sh new file mode 100644 index 000000000..ab1656b58 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,4 @@ +#!/bin/bash -e + +export PYTHONPATH="$PWD/framework/python/src" +python3 -u modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py From 9a74d65ded8b47791a977ef9472ce39318cfd2a4 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 29 Jun 2023 11:43:26 -0600 Subject: [PATCH 09/25] working poc for test containers and grpc client to dhcp-1 --- cmd/start | 90 +++++------ conf/system.json.example | 18 +-- modules/network/base/base.Dockerfile | 58 +++---- modules/network/dhcp-1/conf/dhcpd.conf | 54 +++---- modules/network/dhcp-1/dhcp-1.Dockerfile | 46 +++--- .../dhcp-1/python/src/grpc/dhcp_leases.py | 88 ----------- .../dhcp-1/python/src/grpc/network_service.py | 19 +-- .../dhcp-1/python/src/grpc/proto/grpc.proto | 58 ++++--- modules/test/conn/bin/start_test_module | 15 ++ modules/test/conn/conn.Dockerfile | 27 ++++ .../test/conn/python/src/connection_module.py | 27 ++++ .../conn/python/src/grpc/proto/dhcp/client.py | 98 ++++++++++++ run_tests.sh | 8 +- testing/test_baseline | 144 +++++++++--------- 14 files changed, 423 insertions(+), 327 deletions(-) create mode 100644 modules/test/conn/python/src/grpc/proto/dhcp/client.py diff --git a/cmd/start b/cmd/start index 55d2e52eb..f599f29a8 100755 --- a/cmd/start +++ b/cmd/start @@ -1,46 +1,46 @@ -#!/bin/bash -e - -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo cmd/start" - exit 1 -fi - -# Ensure that /var/run/netns folder exists -mkdir -p /var/run/netns - -# Clear up existing runtime files -rm -rf runtime - -# Check if python modules exist. Install if not -[ ! -d "venv" ] && cmd/install - -# Activate Python virtual environment -source venv/bin/activate - -# TODO: Execute python code -# Set the PYTHONPATH to include the "src" directory -export PYTHONPATH="$PWD/framework/python/src" -python -u framework/python/src/core/test_runner.py $@ - -# TODO: Work in progress code for containerization of OVS module -# asyncRun() { -# "$@" & -# pid="$!" -# echo "PID Running: " $pid -# trap "echo 'Stopping PID $pid'; kill -SIGTERM $pid" SIGINT SIGTERM - -# sleep 10 - -# # A signal emitted while waiting will make the wait command return code > 128 -# # Let's wrap it in a loop that doesn't end before the process is indeed stopped -# while kill -0 $pid > /dev/null 2>&1; do -# #while $(kill -0 $pid 2>/dev/null); do -# wait -# done -# } - -# # -u flag allows python print statements -# # to be logged by docker by running unbuffered -# asyncRun python3 -u python/src/run.py $@ - +#!/bin/bash -e + +if [[ "$EUID" -ne 0 ]]; then + echo "Must run as root. Use sudo cmd/start" + exit 1 +fi + +# Ensure that /var/run/netns folder exists +mkdir -p /var/run/netns + +# Clear up existing runtime files +rm -rf runtime + +# Check if python modules exist. Install if not +[ ! -d "venv" ] && cmd/install + +# Activate Python virtual environment +source venv/bin/activate + +# TODO: Execute python code +# Set the PYTHONPATH to include the "src" directory +export PYTHONPATH="$PWD/framework/python/src" +python -u framework/python/src/core/test_runner.py $@ + +# TODO: Work in progress code for containerization of OVS module +# asyncRun() { +# "$@" & +# pid="$!" +# echo "PID Running: " $pid +# trap "echo 'Stopping PID $pid'; kill -SIGTERM $pid" SIGINT SIGTERM + +# sleep 10 + +# # A signal emitted while waiting will make the wait command return code > 128 +# # Let's wrap it in a loop that doesn't end before the process is indeed stopped +# while kill -0 $pid > /dev/null 2>&1; do +# #while $(kill -0 $pid 2>/dev/null); do +# wait +# done +# } + +# # -u flag allows python print statements +# # to be logged by docker by running unbuffered +# asyncRun python3 -u python/src/run.py $@ + deactivate \ No newline at end of file diff --git a/conf/system.json.example b/conf/system.json.example index ecf480104..e99e013f3 100644 --- a/conf/system.json.example +++ b/conf/system.json.example @@ -1,10 +1,10 @@ -{ - "network": { - "device_intf": "enx123456789123", - "internet_intf": "enx123456789124" - }, - "log_level": "INFO", - "startup_timeout": 60, - "monitor_period": 300, - "runtime": 1200 +{ + "network": { + "device_intf": "enx123456789123", + "internet_intf": "enx123456789124" + }, + "log_level": "INFO", + "startup_timeout": 60, + "monitor_period": 300, + "runtime": 1200 } \ No newline at end of file diff --git a/modules/network/base/base.Dockerfile b/modules/network/base/base.Dockerfile index 3bccb9774..01dc245fd 100644 --- a/modules/network/base/base.Dockerfile +++ b/modules/network/base/base.Dockerfile @@ -1,30 +1,30 @@ -# Image name: test-run/base -FROM ubuntu:jammy - -ARG MODULE_NAME=base -ARG MODULE_DIR=modules/network/$MODULE_NAME -ARG COMMON_DIR=framework/python/src/common - -# Install common software -RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix - -# Install common python modules -COPY $COMMON_DIR/ /testrun/python/src/common - -# Setup the base python requirements -COPY $MODULE_DIR/python /testrun/python - -# Install all python requirements for the module -RUN pip3 install -r /testrun/python/requirements.txt - -# Add the bin files -COPY $MODULE_DIR/bin /testrun/bin - -# Remove incorrect line endings -RUN dos2unix /testrun/bin/* - -# Make sure all the bin files are executable -RUN chmod u+x /testrun/bin/* - -#Start the network module +# Image name: test-run/base +FROM ubuntu:jammy + +ARG MODULE_NAME=base +ARG MODULE_DIR=modules/network/$MODULE_NAME +ARG COMMON_DIR=framework/python/src/common + +# Install common software +RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix + +# Install common python modules +COPY $COMMON_DIR/ /testrun/python/src/common + +# Setup the base python requirements +COPY $MODULE_DIR/python /testrun/python + +# Install all python requirements for the module +RUN pip3 install -r /testrun/python/requirements.txt + +# Add the bin files +COPY $MODULE_DIR/bin /testrun/bin + +# Remove incorrect line endings +RUN dos2unix /testrun/bin/* + +# Make sure all the bin files are executable +RUN chmod u+x /testrun/bin/* + +#Start the network module ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/modules/network/dhcp-1/conf/dhcpd.conf b/modules/network/dhcp-1/conf/dhcpd.conf index 38eeaf693..ee171279c 100644 --- a/modules/network/dhcp-1/conf/dhcpd.conf +++ b/modules/network/dhcp-1/conf/dhcpd.conf @@ -1,28 +1,28 @@ -default-lease-time 300; - -failover peer "failover-peer" { - primary; - address 10.10.10.2; - port 847; - peer address 10.10.10.3; - peer port 647; - max-response-delay 60; - max-unacked-updates 10; - mclt 3600; - split 128; - load balance max seconds 3; -} - -subnet 10.10.10.0 netmask 255.255.255.0 { - option ntp-servers 10.10.10.5; - option subnet-mask 255.255.255.0; - option broadcast-address 10.10.10.255; - option routers 10.10.10.1; - option domain-name-servers 10.10.10.4; - interface veth0; - authoritative; - pool { - failover peer "failover-peer"; - range 10.10.10.10 10.10.10.20; - } +default-lease-time 300; + +failover peer "failover-peer" { + primary; + address 10.10.10.2; + port 847; + peer address 10.10.10.3; + peer port 647; + max-response-delay 60; + max-unacked-updates 10; + mclt 3600; + split 128; + load balance max seconds 3; +} + +subnet 10.10.10.0 netmask 255.255.255.0 { + option ntp-servers 10.10.10.5; + option subnet-mask 255.255.255.0; + option broadcast-address 10.10.10.255; + option routers 10.10.10.1; + option domain-name-servers 10.10.10.4; + interface veth0; + authoritative; + pool { + failover peer "failover-peer"; + range 10.10.10.10 10.10.10.20; + } } \ No newline at end of file diff --git a/modules/network/dhcp-1/dhcp-1.Dockerfile b/modules/network/dhcp-1/dhcp-1.Dockerfile index 22430b90f..f1bdd9ccc 100644 --- a/modules/network/dhcp-1/dhcp-1.Dockerfile +++ b/modules/network/dhcp-1/dhcp-1.Dockerfile @@ -1,23 +1,23 @@ -# Image name: test-run/dhcp-primary -FROM test-run/base:latest - -ARG MODULE_NAME=dhcp-1 -ARG MODULE_DIR=modules/network/$MODULE_NAME - -# Install all necessary packages -RUN apt-get install -y wget - -#Update the oui.txt file from ieee -RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ - -# Install dhcp server -RUN apt-get install -y isc-dhcp-server radvd - -# Copy over all configuration files -COPY $MODULE_DIR/conf /testrun/conf - -# Copy over all binary files -COPY $MODULE_DIR/bin /testrun/bin - -# Copy over all python files -COPY $MODULE_DIR/python /testrun/python +# Image name: test-run/dhcp-primary +FROM test-run/base:latest + +ARG MODULE_NAME=dhcp-1 +ARG MODULE_DIR=modules/network/$MODULE_NAME + +# Install all necessary packages +RUN apt-get install -y wget + +#Update the oui.txt file from ieee +RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ + +# Install dhcp server +RUN apt-get install -y isc-dhcp-server radvd + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py b/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py index 6ea9ce543..a3532c7a4 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py @@ -34,12 +34,6 @@ ] DHCP_CONFIG_FILE = '/etc/dhcp/dhcpd.conf' -reservedHostTemplate=""" -host HOSTNAME{ - hardware ethernet HW_ADDR; - fixed-address RESERVED_IP; -}""" - class DHCPLeases: """Leases for the DHCP server""" @@ -47,82 +41,6 @@ def __init__(self): global LOGGER LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') - def enable_failover(self): - config = self._get_config() - try: - LOGGER.info('Enabling failover peer config block') - failoverStart = config.find('failover peer') - 1 - failoverEnd = config.find('}',failoverStart) - failover = config[failoverStart:failoverEnd+1] - lines = failover.split('\n') - for i in range(len(lines)): - if lines[i].strip().startswith('#'): - lines[i] = lines[i].replace('#','',1) - enabled_config = '\n'.join(lines) - new_config = config[0:failoverStart] + enabled_config + config[failoverEnd+1:] - LOGGER.info('Failover peer config block enabled') - - LOGGER.info('Enabling all failover references') - lines = new_config.split('\n') - for i in range(len(lines)): - if lines[i].strip().startswith('#') and 'failover peer' in lines[i]: - lines[i] = lines[i].replace('#','',1) - LOGGER.info('Failover references enabled') - - new_config = '\n'.join(lines) - self._write_config(new_config) - - except Exception as e: - LOGGER.error("Could not disable failover : " + str(e)) - - def disable_failover(self): - config = self._get_config() - try: - LOGGER.info('Disabling failover peer config block') - failoverStart = config.find('failover peer') - failoverEnd = config.find('}',failoverStart) - failover = config[failoverStart:failoverEnd+1] - lines = failover.split('\n') - for i in range(len(lines)): - if not lines[i].strip().startswith('#'): - lines[i] = '#' + lines[i] - disabled_config = '\n'.join(lines) - new_config = config[0:failoverStart] + disabled_config + config[failoverEnd+1:] - LOGGER.info('Failover peer config block disabled') - - LOGGER.info('Disabling all failover references') - lines = new_config.split('\n') - for i in range(len(lines)): - if lines[i].strip().startswith('failover peer'): - lines[i] = '#' + lines[i] - LOGGER.info('Failover references disabled') - - new_config = '\n'.join(lines) - self._write_config(new_config) - - except Exception as e: - LOGGER.error("Could not disable failover : " + str(e)) - - def add_reserved_host(self,hostname,hw_addr,ip_addr): - self.delete_reserved_host(hw_addr) - LOGGER.info("Add Reserved Host: " + hostname + ":" + hw_addr + ":" + ip_addr) - reservedHost = reservedHostTemplate.replace("HOSTNAME",hostname) - reservedHost = reservedHost.replace("HW_ADDR",hw_addr) - reservedHost = reservedHost.replace("RESERVED_IP",ip_addr) - newConfig = self._get_config() + reservedHost - self._write_config(newConfig) - - def delete_reserved_host(self,hw_addr): - config = self._get_config() - try: - ixHw = config.find("hardware ethernet " + hw_addr) - hostStart = config.rindex("host",0,ixHw) - hostEnd = config.find("}",hostStart) - newConfig = config[0:hostStart] + config[hostEnd+1:] - self._write_config(newConfig) - except Exception as e: - LOGGER.error("Could not delete host: " + hw_addr + str(e)) - def delete_all_hosts(self): LOGGER.info("Deleting hosts") for lease in DHCP_LEASE_FILES: @@ -138,12 +56,6 @@ def delete_all_hosts(self): with open(lease,'w'): pass - def _get_config(self): - content = None - with open(DHCP_CONFIG_FILE,"r") as f: - content= f.read() - return content - def get_lease(self,hw_addr): for lease in self.get_leases(): if lease.hw_addr == hw_addr: diff --git a/modules/network/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc/network_service.py index 38c0e2fae..81ede3cb0 100644 --- a/modules/network/dhcp-1/python/src/grpc/network_service.py +++ b/modules/network/dhcp-1/python/src/grpc/network_service.py @@ -46,7 +46,7 @@ def AddReservedLease(self, request, context): # pylint: disable=W0613 traceback.print_exc() return pb2.Response(code=200, message='{}') - def DeleteReservedLease(self, request, context): # pylint: disable=W0613 + def DeleteReservedLease(self, request, context): # pylint: disable=W0613 print("Delete Reserved Lease Called") try: dhcp_config = self._get_dhcp_config() @@ -80,7 +80,15 @@ def EnableFailover(self, request, contest): # pylint: disable=W0613 print("Failed: " + str(e)) return pb2.Response(code=200, message='{}') - def GetIPAddress(self, request, context): # pylint: disable=W0613 + def GetDHCPRange(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP configuration and return + the first range from the first subnet in the file + """ + pool = self._get_dhcp_config()._subnets[0].pools[0] + return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + + def GetLease(self, request, context): # pylint: disable=W0613 """ Resolve the current DHCP leased address for the provided MAC address @@ -91,13 +99,6 @@ def GetIPAddress(self, request, context): # pylint: disable=W0613 else: return pb2.Response(code=200, message='{}') - def GetDHCPRange(self, request, context): # pylint: disable=W0613 - """ - Resolve the current DHCP configuration and return - the first range from the first subnet in the file - """ - pool = self.get_dhcp_config().subnets[0].pools[0] - return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) def SetDHCPRange(self, request, context): # pylint: disable=W0613 """ diff --git a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto index 79749d8a4..b5af4e9ec 100644 --- a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto +++ b/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto @@ -2,22 +2,49 @@ syntax = "proto3"; service NetworkModule { + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; + + rpc DeleteReservedLease(DeleteReservedLeaseRequest) returns (Response) {}; + + rpc DisableFailover(DisableFailoverRequest) returns (Response) {}; + + rpc EnableFailover(EnableFailoverRequest) returns (Response) {}; + rpc GetDHCPRange(GetDHCPRangeRequest) returns (DHCPRange) {}; - rpc SetDHCPRange(DHCPRange) returns (Response) {}; + rpc GetLease(GetLeaseRequest) returns (Response) {}; rpc GetStatus(GetStatusRequest) returns (Response) {}; - rpc GetIPAddress(GetIPAddressRequest) returns (Response) {}; + rpc SetDHCPRange(SetDHCPRangeRequest) returns (Response) {}; +} + +message AddReservedLeaseRequest { + string hostname = 1; + string hw_addr = 2; + string ip_addr = 3; +} + +message DeleteReservedLeaseRequest { + string hw_addr = 1; +} - rpc SetLeaseAddress(SetLeaseAddressRequest) returns (Response) {}; +message DisableFailoverRequest {} - rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; +message EnableFailoverRequest {} - rpc DisableFailover(DisableFailoverRequest) returns (Response) {}; +message GetDHCPRangeRequest {} - rpc EnableFailover(EnableFailoverRequest) returns (Response) {}; +message GetLeaseRequest { + string hw_addr = 1; +} + +message GetStatusRequest {} +message SetDHCPRangeRequest { + int32 code = 1; + string start = 2; + string end = 3; } message Response { @@ -31,24 +58,13 @@ message DHCPRange { string end = 3; } -message AddReservedLeaseRequest { - string hostname = 1; - string hw_addr = 2; - string ip_addr = 3; -} -message GetDHCPRangeRequest {} -message GetIPAddressRequest { - string hw_addr = 1; -} -message GetStatusRequest {} -message DisableFailoverRequest {} -message EnableFailoverRequest {} -message SetLeaseAddressRequest { - string ipAddress = 1; -} \ No newline at end of file + + + + diff --git a/modules/test/conn/bin/start_test_module b/modules/test/conn/bin/start_test_module index 8290c0764..0f6f490df 100644 --- a/modules/test/conn/bin/start_test_module +++ b/modules/test/conn/bin/start_test_module @@ -31,6 +31,21 @@ touch $RESULT_FILE chown $HOST_USER $LOG_FILE chown $HOST_USER $RESULT_FILE +echo "Python SRC" +ls -al /testrun/python/src + +echo "gRPC SRC" +ls -al /testrun/python/src/grpc + +echo "Proto SRC" +ls -al /testrun/python/src/grpc/proto + +echo "DHCP Src" +ls -al /testrun/python/src/grpc/proto/dhcp + +echo "PYTHON_PATH: $PYTHONPATH" + + # Run the python scrip that will execute the tests for this module # -u flag allows python print statements # to be logged by docker by running unbuffered diff --git a/modules/test/conn/conn.Dockerfile b/modules/test/conn/conn.Dockerfile index 2526b0046..a3dc36e06 100644 --- a/modules/test/conn/conn.Dockerfile +++ b/modules/test/conn/conn.Dockerfile @@ -3,6 +3,8 @@ FROM test-run/base-test:latest ARG MODULE_NAME=conn ARG MODULE_DIR=modules/test/$MODULE_NAME +ARG GRPC_PROTO_DIR=/testrun/python/src/grpc/proto/dhcp +ARG GRPC_PROTO_FILE="grpc.proto" # Install all necessary packages RUN apt-get install -y wget @@ -24,3 +26,28 @@ COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files COPY $MODULE_DIR/python /testrun/python + + +# Set the PYTHONPATH to include the "src" directory +ARG ROOT_SRC_DIR="/testrun/python/src" +ARG GRPC_SRC_DIR="/testrun/python/src/grpc" +ARG PROTO_GRPC_SRC_DIR="/testrun/python/src/grpc/proto" +ARG DHCP_GRPC_SRC_DIR="/testrun/python/src/grpc/proto/dhcp" + +ENV PYTHONPATH=$ROOT_SRC_DIR:$GRPC_SRC_DIR:$PROTO_GRPC_SRC_DIR:$DHCP_GRPC_SRC_DIR + +ARG GRPC_PROTO_DIR="proto/dhcp" +ARG GRPC_PROTO_FILE="grpc.proto" + +# Copy over the required network module grpc proto files +COPY modules/network/dhcp-1/python/src/grpc/proto/grpc.proto testrun/python/src/grpc/proto/dhcp/ + + +# Move into the grpc directory +WORKDIR $GRPC_SRC_DIR + +# Build the grpc proto file every time before starting server +RUN python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. + +WORKDIR .. + diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index 196c335d8..c63ed906e 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -17,6 +17,7 @@ import sys from scapy.all import * from test_module import TestModule +from dhcp.client import Client as DHCPClient LOG_NAME = "test_connection" LOGGER = None @@ -33,6 +34,32 @@ def __init__(self, module): super().__init__(module_name=module, log_name=LOG_NAME) global LOGGER LOGGER = self._get_logger() + self.client = DHCPClient() + + response = self.client.add_reserved_lease('test','00:11:22:33:44:55','10.10.10.21') + print("AddLeaseResp: " + str(response)) + + response = self.client.delete_reserved_lease('00:11:22:33:44:55') + print("DelLeaseResp: " + str(response)) + + response = self.client.disable_failover() + print("FailoverDisabled: " + str(response)) + + response = self.client.enable_failover() + print("FailoverEnabled: " + str(response)) + + response = self.client.get_dhcp_range() + print("DHCP Range: " + str(response)) + + response = self.client.get_lease(self._device_mac) + print("Lease: " + str(response)) + + response = self.client.get_status() + print("Status: " + str(response)) + + response = self.client.set_dhcp_range('10.10.10.20','10.10.10.30') + print("Set Range: " + str(response)) + def _connection_mac_address(self): LOGGER.info("Running connection.mac_address") diff --git a/modules/test/conn/python/src/grpc/proto/dhcp/client.py b/modules/test/conn/python/src/grpc/proto/dhcp/client.py new file mode 100644 index 000000000..921929edb --- /dev/null +++ b/modules/test/conn/python/src/grpc/proto/dhcp/client.py @@ -0,0 +1,98 @@ +import grpc +import grpc_pb2_grpc as pb2_grpc +import grpc_pb2 as pb2 + +DEFAULT_PORT = '5001' +DEFAULT_HOST = '10.10.10.2' # Default DHCP1 server + + +class Client(): + + def __init__(self, port=DEFAULT_PORT, host=DEFAULT_HOST): + self._port = port + self._host = host + + # Create a gRPC channel to connect to the server + self._channel = grpc.insecure_channel(self._host + ':' + self._port) + + # Create a gRPC stub + self._stub = pb2_grpc.NetworkModuleStub(self._channel) + + def add_reserved_lease(self, hostname, hw_addr, ip_addr): + # Create a request message + request = pb2.AddReservedLeaseRequest() + request.hostname = hostname + request.hw_addr = hw_addr + request.ip_addr = ip_addr + + # Make the RPC call + response = self._stub.AddReservedLease(request) + + return response + + def delete_reserved_lease(self, hw_addr): + # Create a request message + request = pb2.DeleteReservedLeaseRequest() + request.hw_addr = hw_addr + + # Make the RPC call + response = self._stub.DeleteReservedLease(request) + + return response + + def disable_failover(self): + # Create a request message + request = pb2.DisableFailoverRequest() + + # Make the RPC call + response = self._stub.DisableFailover(request) + + return response + + def enable_failover(self): + # Create a request message + request = pb2.EnableFailoverRequest() + + # Make the RPC call + response = self._stub.EnableFailover(request) + + return response + + def get_dhcp_range(self): + # Create a request message + request = pb2.GetDHCPRangeRequest() + + # Make the RPC call + response = self._stub.GetDHCPRange(request) + + return response + + def get_lease(self,hw_addr): + # Create a request message + request = pb2.GetLeaseRequest() + request.hw_addr=hw_addr + + # Make the RPC call + response = self._stub.GetLease(request) + + return response + + def get_status(self): + # Create a request message + request = pb2.GetStatusRequest() + + # Make the RPC call + response = self._stub.GetStatus(request) + + return response + + def set_dhcp_range(self,start,end): + # Create a request message + request = pb2.SetDHCPRangeRequest() + request.start=start + request.end=end + + # Make the RPC call + response = self._stub.SetDHCPRange(request) + + return response diff --git a/run_tests.sh b/run_tests.sh index ab1656b58..017d5f6a8 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e - -export PYTHONPATH="$PWD/framework/python/src" -python3 -u modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py +#!/bin/bash -e + +export PYTHONPATH="$PWD/framework/python/src" +python3 -u modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py diff --git a/testing/test_baseline b/testing/test_baseline index bf191b88f..5dff50718 100755 --- a/testing/test_baseline +++ b/testing/test_baseline @@ -1,72 +1,72 @@ - -#!/bin/bash -e - -TESTRUN_OUT=/tmp/testrun.log - -ifconfig - -# Setup requirements -sudo apt-get update -sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils - -pip3 install pytest - -# Setup device network -sudo ip link add dev endev0a type veth peer name endev0b -sudo ip link set dev endev0a up -sudo ip link set dev endev0b up -sudo docker network create -d macvlan -o parent=endev0b endev0 - -# Start OVS -sudo /usr/share/openvswitch/scripts/ovs-ctl start - -# Build Test Container -sudo docker build ./testing/docker/ci_baseline -t ci1 -f ./testing/docker/ci_baseline/Dockerfile - -cat <conf/system.json -{ - "network": { - "device_intf": "endev0a", - "internet_intf": "eth0" - }, - "log_level": "DEBUG" -} -EOF - -sudo cmd/install - -sudo cmd/start --single-intf > $TESTRUN_OUT 2>&1 & -TPID=$! - -# Time to wait for testrun to be ready -WAITING=600 -for i in `seq 1 $WAITING`; do - if [[ -n $(fgrep "Waiting for devices on the network" $TESTRUN_OUT) ]]; then - break - fi - - if [[ ! -d /proc/$TPID ]]; then - cat $TESTRUN_OUT - echo "error encountered starting test run" - exit 1 - fi - - sleep 1 -done - -if [[ $i -eq $WAITING ]]; then - cat $TESTRUN_OUT - echo "failed after waiting $WAITING seconds for test-run start" - exit 1 -fi - -# Load Test Container -sudo docker run --network=endev0 --cap-add=NET_ADMIN -v /tmp:/out --privileged ci1 - -echo "Done baseline test" - -more $TESTRUN_OUT - -pytest testing/ - -exit $? + +#!/bin/bash -e + +TESTRUN_OUT=/tmp/testrun.log + +ifconfig + +# Setup requirements +sudo apt-get update +sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils + +pip3 install pytest + +# Setup device network +sudo ip link add dev endev0a type veth peer name endev0b +sudo ip link set dev endev0a up +sudo ip link set dev endev0b up +sudo docker network create -d macvlan -o parent=endev0b endev0 + +# Start OVS +sudo /usr/share/openvswitch/scripts/ovs-ctl start + +# Build Test Container +sudo docker build ./testing/docker/ci_baseline -t ci1 -f ./testing/docker/ci_baseline/Dockerfile + +cat <conf/system.json +{ + "network": { + "device_intf": "endev0a", + "internet_intf": "eth0" + }, + "log_level": "DEBUG" +} +EOF + +sudo cmd/install + +sudo cmd/start --single-intf > $TESTRUN_OUT 2>&1 & +TPID=$! + +# Time to wait for testrun to be ready +WAITING=600 +for i in `seq 1 $WAITING`; do + if [[ -n $(fgrep "Waiting for devices on the network" $TESTRUN_OUT) ]]; then + break + fi + + if [[ ! -d /proc/$TPID ]]; then + cat $TESTRUN_OUT + echo "error encountered starting test run" + exit 1 + fi + + sleep 1 +done + +if [[ $i -eq $WAITING ]]; then + cat $TESTRUN_OUT + echo "failed after waiting $WAITING seconds for test-run start" + exit 1 +fi + +# Load Test Container +sudo docker run --network=endev0 --cap-add=NET_ADMIN -v /tmp:/out --privileged ci1 + +echo "Done baseline test" + +more $TESTRUN_OUT + +pytest testing/ + +exit $? From d4424212aeb7dc2225f06b4806b80c0f976cd256 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 29 Jun 2023 14:12:33 -0600 Subject: [PATCH 10/25] Move grpc client code into base image --- modules/test/base/base.Dockerfile | 32 +++++++++++++++++++ .../python/src/grpc/proto/dhcp1}/client.py | 0 modules/test/conn/bin/start_test_module | 7 ++-- modules/test/conn/conn.Dockerfile | 27 +--------------- .../test/conn/python/src/connection_module.py | 20 ++++++------ 5 files changed, 48 insertions(+), 38 deletions(-) rename modules/test/{conn/python/src/grpc/proto/dhcp => base/python/src/grpc/proto/dhcp1}/client.py (100%) diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index b8398eae9..3439a55f6 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -22,5 +22,37 @@ RUN dos2unix /testrun/bin/* # Make sure all the bin files are executable RUN chmod u+x /testrun/bin/* +# Copy over all network module gRPC proto files +ARG NET_MODULE_DIR=modules/network +ARG NET_MODULE_PROTO_DIR=python/src/grpc/proto/grpc.proto +ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc/proto + +COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ +COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ + +# Build all the gRPC proto files +ARG GRPC_DIR="/testrun/python/src/grpc" +ARG GRPC_PROTO_DIR="proto" +ARG GRPC_PROTO_FILE="grpc.proto" + +# Move into the grpc directory +WORKDIR $GRPC_DIR + +# Build the grpc proto file every time before starting server +RUN python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/dhcp1/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. + +RUN python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/dhcp2/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. + +WORKDIR .. + +# Set the PYTHONPATH to include all the directories +ARG ROOT_SRC_DIR="/testrun/python/src" +ARG GRPC_SRC_DIR="/testrun/python/src/grpc" +ARG PROTO_GRPC_SRC_DIR="/testrun/python/src/grpc/proto" +ARG DHCP1_GRPC_SRC_DIR="/testrun/python/src/grpc/proto/dhcp1" +ARG DHCP2_GRPC_SRC_DIR="/testrun/python/src/grpc/proto/dhcp2" + +ENV PYTHONPATH=$ROOT_SRC_DIR:$GRPC_SRC_DIR:$PROTO_GRPC_SRC_DIR:$DHCP1_GRPC_SRC_DIR:$DHCP2_GRPC_SRC_DIR + # Start the test module ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/modules/test/conn/python/src/grpc/proto/dhcp/client.py b/modules/test/base/python/src/grpc/proto/dhcp1/client.py similarity index 100% rename from modules/test/conn/python/src/grpc/proto/dhcp/client.py rename to modules/test/base/python/src/grpc/proto/dhcp1/client.py diff --git a/modules/test/conn/bin/start_test_module b/modules/test/conn/bin/start_test_module index 0f6f490df..6bc033e79 100644 --- a/modules/test/conn/bin/start_test_module +++ b/modules/test/conn/bin/start_test_module @@ -40,8 +40,11 @@ ls -al /testrun/python/src/grpc echo "Proto SRC" ls -al /testrun/python/src/grpc/proto -echo "DHCP Src" -ls -al /testrun/python/src/grpc/proto/dhcp +echo "DHCP1 Src" +ls -al /testrun/python/src/grpc/proto/dhcp1 + +echo "DHCP2 Src" +ls -al /testrun/python/src/grpc/proto/dhcp2 echo "PYTHON_PATH: $PYTHONPATH" diff --git a/modules/test/conn/conn.Dockerfile b/modules/test/conn/conn.Dockerfile index a3dc36e06..9e62408fe 100644 --- a/modules/test/conn/conn.Dockerfile +++ b/modules/test/conn/conn.Dockerfile @@ -25,29 +25,4 @@ COPY $MODULE_DIR/conf /testrun/conf COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY $MODULE_DIR/python /testrun/python - - -# Set the PYTHONPATH to include the "src" directory -ARG ROOT_SRC_DIR="/testrun/python/src" -ARG GRPC_SRC_DIR="/testrun/python/src/grpc" -ARG PROTO_GRPC_SRC_DIR="/testrun/python/src/grpc/proto" -ARG DHCP_GRPC_SRC_DIR="/testrun/python/src/grpc/proto/dhcp" - -ENV PYTHONPATH=$ROOT_SRC_DIR:$GRPC_SRC_DIR:$PROTO_GRPC_SRC_DIR:$DHCP_GRPC_SRC_DIR - -ARG GRPC_PROTO_DIR="proto/dhcp" -ARG GRPC_PROTO_FILE="grpc.proto" - -# Copy over the required network module grpc proto files -COPY modules/network/dhcp-1/python/src/grpc/proto/grpc.proto testrun/python/src/grpc/proto/dhcp/ - - -# Move into the grpc directory -WORKDIR $GRPC_SRC_DIR - -# Build the grpc proto file every time before starting server -RUN python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. - -WORKDIR .. - +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index c63ed906e..c2600335b 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -17,7 +17,7 @@ import sys from scapy.all import * from test_module import TestModule -from dhcp.client import Client as DHCPClient +from dhcp1.client import Client as DHCPClient1 LOG_NAME = "test_connection" LOGGER = None @@ -34,30 +34,30 @@ def __init__(self, module): super().__init__(module_name=module, log_name=LOG_NAME) global LOGGER LOGGER = self._get_logger() - self.client = DHCPClient() + self.dhcp1_client = DHCPClient1() - response = self.client.add_reserved_lease('test','00:11:22:33:44:55','10.10.10.21') + response = self.dhcp1_client.add_reserved_lease('test','00:11:22:33:44:55','10.10.10.21') print("AddLeaseResp: " + str(response)) - response = self.client.delete_reserved_lease('00:11:22:33:44:55') + response = self.dhcp1_client.delete_reserved_lease('00:11:22:33:44:55') print("DelLeaseResp: " + str(response)) - response = self.client.disable_failover() + response = self.dhcp1_client.disable_failover() print("FailoverDisabled: " + str(response)) - response = self.client.enable_failover() + response = self.dhcp1_client.enable_failover() print("FailoverEnabled: " + str(response)) - response = self.client.get_dhcp_range() + response = self.dhcp1_client.get_dhcp_range() print("DHCP Range: " + str(response)) - response = self.client.get_lease(self._device_mac) + response = self.dhcp1_client.get_lease(self._device_mac) print("Lease: " + str(response)) - response = self.client.get_status() + response = self.dhcp1_client.get_status() print("Status: " + str(response)) - response = self.client.set_dhcp_range('10.10.10.20','10.10.10.30') + response = self.dhcp1_client.set_dhcp_range('10.10.10.20','10.10.10.30') print("Set Range: " + str(response)) From ec7cc659e016ab4ffbf7b7c7783854b7488c4870 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 29 Jun 2023 15:54:31 -0600 Subject: [PATCH 11/25] Move grpc proto builds outside of dockerfile into module startup script --- modules/test/base/base.Dockerfile | 15 ----------- modules/test/base/bin/setup_grpc_clients | 34 ++++++++++++++++++++++++ modules/test/base/bin/start_module | 2 ++ 3 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 modules/test/base/bin/setup_grpc_clients diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index 3439a55f6..960c76ed1 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -30,21 +30,6 @@ ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc/proto COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ -# Build all the gRPC proto files -ARG GRPC_DIR="/testrun/python/src/grpc" -ARG GRPC_PROTO_DIR="proto" -ARG GRPC_PROTO_FILE="grpc.proto" - -# Move into the grpc directory -WORKDIR $GRPC_DIR - -# Build the grpc proto file every time before starting server -RUN python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/dhcp1/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. - -RUN python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/dhcp2/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. - -WORKDIR .. - # Set the PYTHONPATH to include all the directories ARG ROOT_SRC_DIR="/testrun/python/src" ARG GRPC_SRC_DIR="/testrun/python/src/grpc" diff --git a/modules/test/base/bin/setup_grpc_clients b/modules/test/base/bin/setup_grpc_clients new file mode 100644 index 000000000..8cc003cfa --- /dev/null +++ b/modules/test/base/bin/setup_grpc_clients @@ -0,0 +1,34 @@ +#!/bin/bash -e + +GRPC_DIR="/testrun/python/src/grpc" +GRPC_PROTO_DIR="proto" +GRPC_PROTO_FILE="grpc.proto" + +# Build the grpc proto file +build_grpc_client(){ + MODULE=$1 + echo "Building gRPC proto: $MODULE" + python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$MODULE/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. +} + +# Build the grpc proto files for every module that has a proto defined +build_grpc_clients(){ + + for dir in "$GRPC_DIR/$GRPC_PROTO_DIR"/*/;do + if [ -f $dir/$GRPC_PROTO_FILE ];then + # Extract the last folder name + last_folder="${dir%%/}" + last_folder="${last_folder##*/}" + build_grpc_client "$last_folder" + fi + done +} + +# Move into the grpc directory. +# This is necessary to build the proto files +# with the correct import paths +pushd $GRPC_DIR >/dev/null 2>&1 + +build_grpc_clients + +popd >/dev/null 2>&1 \ No newline at end of file diff --git a/modules/test/base/bin/start_module b/modules/test/base/bin/start_module index 3e4737d8b..f45254e37 100644 --- a/modules/test/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -43,6 +43,8 @@ then exit 1 fi +$BIN_DIR/setup_grpc_clients + echo "Starting module $MODULE_NAME..." $BIN_DIR/setup_binaries $BIN_DIR From e3102ca027c0fe036b921711de16474cee85a4b1 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 30 Jun 2023 09:37:06 -0600 Subject: [PATCH 12/25] Setup pythonpath var in test module base startup process misc cleanup --- modules/test/base/base.Dockerfile | 9 --------- modules/test/base/bin/setup_python_path | 25 +++++++++++++++++++++++++ modules/test/base/bin/start_module | 13 +++++++++++-- modules/test/conn/bin/start_test_module | 18 ------------------ 4 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 modules/test/base/bin/setup_python_path diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index 960c76ed1..e12c34b36 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -30,14 +30,5 @@ ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc/proto COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ -# Set the PYTHONPATH to include all the directories -ARG ROOT_SRC_DIR="/testrun/python/src" -ARG GRPC_SRC_DIR="/testrun/python/src/grpc" -ARG PROTO_GRPC_SRC_DIR="/testrun/python/src/grpc/proto" -ARG DHCP1_GRPC_SRC_DIR="/testrun/python/src/grpc/proto/dhcp1" -ARG DHCP2_GRPC_SRC_DIR="/testrun/python/src/grpc/proto/dhcp2" - -ENV PYTHONPATH=$ROOT_SRC_DIR:$GRPC_SRC_DIR:$PROTO_GRPC_SRC_DIR:$DHCP1_GRPC_SRC_DIR:$DHCP2_GRPC_SRC_DIR - # Start the test module ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/modules/test/base/bin/setup_python_path b/modules/test/base/bin/setup_python_path new file mode 100644 index 000000000..8201bbb36 --- /dev/null +++ b/modules/test/base/bin/setup_python_path @@ -0,0 +1,25 @@ +#!/bin/bash + +ROOT_DIRECTORY="/testrun/python/src" + +# Function to recursively add subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath() { + local directory=$1 + local subdirectories=( "$directory"/* ) + local subdirectory + + for subdirectory in "${subdirectories[@]}"; do + if [ -d "$subdirectory" ]; then + export PYTHONPATH="$PYTHONPATH:$subdirectory" + add_subdirectories_to_pythonpath "$subdirectory" + fi + done +} + +# Set PYTHONPATH initially to an empty string +export PYTHONPATH="" + +# Add all subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath "$ROOT_DIRECTORY" + +echo "$PYTHONPATH" \ No newline at end of file diff --git a/modules/test/base/bin/start_module b/modules/test/base/bin/start_module index f45254e37..fff724323 100644 --- a/modules/test/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -43,12 +43,21 @@ then exit 1 fi +echo "Setting up PYTHONPATH..." +# Setup the PYTHONPATH so all imports work as expected +export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" + +# Build all gRPC files from the proto for use in +# gRPC clients for communications to network modules +echo "Building gRPC files from available proto files..." $BIN_DIR/setup_grpc_clients -echo "Starting module $MODULE_NAME..." - +echo "Configuring binary files..." $BIN_DIR/setup_binaries $BIN_DIR +echo "Starting module $MODULE_NAME..." + # Only start network services if the test container needs # a network connection to run its tests if [ $NETWORK_REQUIRED == "true" ];then diff --git a/modules/test/conn/bin/start_test_module b/modules/test/conn/bin/start_test_module index 6bc033e79..8290c0764 100644 --- a/modules/test/conn/bin/start_test_module +++ b/modules/test/conn/bin/start_test_module @@ -31,24 +31,6 @@ touch $RESULT_FILE chown $HOST_USER $LOG_FILE chown $HOST_USER $RESULT_FILE -echo "Python SRC" -ls -al /testrun/python/src - -echo "gRPC SRC" -ls -al /testrun/python/src/grpc - -echo "Proto SRC" -ls -al /testrun/python/src/grpc/proto - -echo "DHCP1 Src" -ls -al /testrun/python/src/grpc/proto/dhcp1 - -echo "DHCP2 Src" -ls -al /testrun/python/src/grpc/proto/dhcp2 - -echo "PYTHON_PATH: $PYTHONPATH" - - # Run the python scrip that will execute the tests for this module # -u flag allows python print statements # to be logged by docker by running unbuffered From ed6257f10046d6b6a77d4c59d90ace4b05db9d47 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 30 Jun 2023 11:41:50 -0600 Subject: [PATCH 13/25] pylinting and logging updates --- .../dhcp-1/python/src/grpc/dhcp_config.py | 73 ++++++------ .../dhcp-1/python/src/grpc/dhcp_lease.py | 29 ++--- .../dhcp-1/python/src/grpc/dhcp_leases.py | 68 +++++------ .../dhcp-1/python/src/grpc/network_service.py | 110 +++++++++++------- run_tests.sh | 8 +- 5 files changed, 158 insertions(+), 130 deletions(-) diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py index af5350a06..83f3cbe5a 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py @@ -11,13 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Contains all the necessary classes to maintain the DHCP server's configuration""" import re import os import sys + + # Add the parent directory to sys.path current_dir = os.path.dirname(os.path.abspath(__file__)) parent_dir = os.path.dirname(current_dir) @@ -25,7 +26,7 @@ import common.logger as logger -LOG_NAME = "dhcp_config" +LOG_NAME = 'dhcp_config' LOGGER = None CONFIG_FILE = '/etc/dhcp/dhcpd.conf' @@ -33,13 +34,6 @@ DEFAULT_LEASE_TIME_KEY = 'default-lease-time' -RESERVED_HOST_TEMPLATE = """ -host {HOSTNAME}{{ - hardware ethernet {HW_ADDR}; - fixed-address {RESERVED_IP}; -}}""" - - class DHCPConfig: """Represents the DHCP Servers configuration and gives access to modify it""" @@ -52,12 +46,14 @@ def __init__(self): LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') def add_reserved_host(self, hostname, hw_addr, ip_addr): - host = DHCPReservedHost(host=hostname,hw_addr=hw_addr,fixed_addr=ip_addr) + host = DHCPReservedHost(hostname=hostname, + hw_addr=hw_addr, + fixed_addr=ip_addr) self._reserved_hosts.append(host) def delete_reserved_host(self, hw_addr): for host in self._reserved_hosts: - if hw_addr == host._hw_addr: + if hw_addr == host.hw_addr: self._reserved_hosts.remove(host) def disable_failover(self): @@ -70,12 +66,12 @@ def enable_failover(self): for subnet in self._subnets: subnet.enable_peer() - def get_reserved_host(self,hw_addr): + def get_reserved_host(self, hw_addr): for host in self._reserved_hosts: - if hw_addr == host._hw_addr: + if hw_addr == host.hw_addr: return host - def write_config(self,config=None): + def write_config(self, config=None): if config is None: conf = str(self) with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: @@ -86,17 +82,17 @@ def write_config(self,config=None): def _get_config(self, config_file=CONFIG_FILE): content = None - with open(config_file, "r") as f: + with open(config_file, 'r', encoding='UTF-8') as f: content = f.read() return content - def make(self,conf): + def make(self, conf): try: self._subnets = self.resolve_subnets(conf) self._peer = DHCPFailoverPeer(conf) self._reserved_hosts = self.resolve_reserved_hosts(conf) - except Exception as e: - print("Failed to make DHCPConfig: " + str(e)) + except Exception as e: # pylint: disable=W0718 + print('Failed to make DHCPConfig: ' + str(e)) def resolve_config(self, config_file=CONFIG_FILE): try: @@ -104,8 +100,8 @@ def resolve_config(self, config_file=CONFIG_FILE): self._subnets = self.resolve_subnets(conf) self._peer = DHCPFailoverPeer(conf) self._reserved_hosts = self.resolve_reserved_hosts(conf) - except Exception as e: - print("Failed to resolve config: " + str(e)) + except Exception as e: # pylint: disable=W0718 + print('Failed to resolve config: ' + str(e)) def resolve_subnets(self, conf): subnets = [] @@ -116,18 +112,18 @@ def resolve_subnets(self, conf): subnets.append(dhcp_subnet) return subnets - def resolve_reserved_hosts(self,conf): + def resolve_reserved_hosts(self, conf): hosts = [] host_start = 0 while True: - host_start = conf.find('host',host_start) + host_start = conf.find('host', host_start) if host_start < 0: break else: - host_end = conf.find('}',host_start) - host = DHCPReservedHost(config=conf[host_start:host_end+1]) + host_end = conf.find('}', host_start) + host = DHCPReservedHost(config=conf[host_start:host_end + 1]) hosts.append(host) - host_start = host_end+1 + host_start = host_end + 1 return hosts def set_range(self, start, end, subnet=0, pool=0): @@ -434,7 +430,7 @@ def __str__(self): if not self._peer_enabled: config = config.replace(FAILOVER_KEY, '#' + FAILOVER_KEY) - + return config def disable_peer(self): @@ -463,14 +459,15 @@ def resolve_pool(self, pool): HARDWARE_KEY = 'hardware ethernet' FIXED_ADDRESS_KEY = 'fixed-address' + class DHCPReservedHost: """Represents a DHCP Servers subnet pool configuration""" - def __init__(self, host=None,hw_addr=None,fixed_addr=None,config=None): + def __init__(self, hostname=None, hw_addr=None, fixed_addr=None, config=None): if config is None: - self._host = host - self._hw_addr = hw_addr - self._fixed_addr = fixed_addr + self.host = hostname + self.hw_addr = hw_addr + self.fixed_addr = fixed_addr else: self.resolve_host(config) @@ -484,11 +481,11 @@ def __str__(self): config = config.format( length='multi-line', HOST_KEY=HOST_KEY, - HOSTNAME=self._host, + HOSTNAME=self.host, HARDWARE_KEY=HARDWARE_KEY, - HW_ADDR=self._hw_addr, + HW_ADDR=self.hw_addr, FIXED_ADDRESS_KEY=FIXED_ADDRESS_KEY, - RESERVED_IP=self._fixed_addr, + RESERVED_IP=self.fixed_addr, ) return config @@ -496,11 +493,9 @@ def resolve_host(self, reserved_host): host_parts = reserved_host.split('\n') for part in host_parts: if HOST_KEY in part: - self._host = part.strip().split(HOST_KEY)[1].strip().split( - '{')[0] + self.host = part.strip().split(HOST_KEY)[1].strip().split('{')[0] elif HARDWARE_KEY in part: - self._hw_addr = part.strip().split(HARDWARE_KEY)[1].strip().split( - ';')[0] + self.hw_addr = part.strip().split(HARDWARE_KEY)[1].strip().split(';')[0] elif FIXED_ADDRESS_KEY in part: - self._fixed_addr = part.strip().split(FIXED_ADDRESS_KEY)[1].strip().split( - ';')[0] + self.fixed_addr = part.strip().split( + FIXED_ADDRESS_KEY)[1].strip().split(';')[0] diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py b/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py index aa47745c6..0d2f43e3b 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py @@ -11,14 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +"""Contains all the necessary methods to create and monitor DHCP +leases on the server""" from datetime import datetime import time -timeFormat = "%Y-%m-%d %H:%M:%S" +time_format = '%Y-%m-%d %H:%M:%S' class DHCPLease(object): + """Represents a DHCP Server lease""" hw_addr = None ip = None hostname = None @@ -29,30 +31,29 @@ def __init__(self, lease): def _make_lease(self, lease): if lease is not None: - sectionsRaw = lease.split(" ") + sections_raw = lease.split(' ') sections = [] - for section in sectionsRaw: - if not (not section.strip()): + for section in sections_raw: + if section.strip(): sections.append(section) self.hw_addr = sections[0] self.ip = sections[1] self.hostname = sections[2] - self.expires = sections[3] + " " + sections[4] - self.manufacturer = " ".join(sections[5:]) - + self.expires = sections[3] + '' '' + sections[4] + self.manufacturer = ' '.join(sections[5:]) def get_millis(self, timestamp): - dt_obj = datetime.strptime(timestamp, timeFormat) + dt_obj = datetime.strptime(timestamp, time_format) millis = dt_obj.timestamp() * 1000 return millis def get_expires_millis(self): return self.get_millis(self.expires) - def isExpired(self): - expiresMilis = self.get_expires_millis() - curTime = int(round(time.time()) * 1000) - return curTime >= expiresMilis + def is_expired(self): + expires_millis = self.get_expires_millis() + cur_time = int(round(time.time()) * 1000) + return cur_time >= expires_millis def __str__(self): lease = {} @@ -71,4 +72,4 @@ def __str__(self): if self.manufacturer is not None: lease['manufacturer'] = self.manufacturer - return str(lease) \ No newline at end of file + return str(lease) diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py b/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py index a3532c7a4..fbfeb7b6c 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Used to resolve the DHCP servers lease information""" - import os import sys from dhcp_lease import DHCPLease @@ -25,15 +24,16 @@ import logger from common import util -LOG_NAME = "dhcp_lease" +LOG_NAME = 'dhcp_lease' LOGGER = None DHCP_LEASE_FILES = [ - "/var/lib/dhcp/dhcpd.leases", "/var/lib/dhcp/dhcpd.leases~", - "/var/lib/dhcp/dhcpd6.leases", "/var/lib/dhcp/dhcpd6.leases~" + '/var/lib/dhcp/dhcpd.leases', '/var/lib/dhcp/dhcpd.leases~', + '/var/lib/dhcp/dhcpd6.leases', '/var/lib/dhcp/dhcpd6.leases~' ] DHCP_CONFIG_FILE = '/etc/dhcp/dhcpd.conf' + class DHCPLeases: """Leases for the DHCP server""" @@ -42,21 +42,21 @@ def __init__(self): LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') def delete_all_hosts(self): - LOGGER.info("Deleting hosts") + LOGGER.info('Deleting hosts') for lease in DHCP_LEASE_FILES: - LOGGER.info("Checking file: " + lease) + LOGGER.info('Checking file: ' + lease) if os.path.exists(lease): - LOGGER.info("File Exists: " + lease) + LOGGER.info('File Exists: ' + lease) try: # Delete existing lease file os.remove(lease) except OSError as e: - LOGGER.info(f"Error occurred while deleting the file: {e}") + LOGGER.info(f'Error occurred while deleting the file: {e}') # Create an empty lease file - with open(lease,'w'): + with open(lease, 'w', encoding='UTF-8'): pass - def get_lease(self,hw_addr): + def get_lease(self, hw_addr): for lease in self.get_leases(): if lease.hw_addr == hw_addr: return lease @@ -64,7 +64,7 @@ def get_lease(self,hw_addr): def get_leases(self): leases = [] lease_list_raw = self._get_lease_list() - LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + "\n") + LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + '\n') lines = lease_list_raw.split( '===============================================================================================' )[1].split('\n') @@ -72,42 +72,42 @@ def get_leases(self): try: lease = DHCPLease(line) leases.append(lease) - except Exception as e: + except Exception as e: # pylint: disable=W0718 # Let non lease lines file without extra checks - LOGGER.error("Making Lease Error: " + str(e)) - LOGGER.error("Not a valid lease line: " + line) + LOGGER.error('Making Lease Error: ' + str(e)) + LOGGER.error('Not a valid lease line: ' + line) return leases - def delete_lease(self,ip_addr): - LOGGER.info("Deleting lease") + def delete_lease(self, ip_addr): + LOGGER.info('Deleting lease') for lease in DHCP_LEASE_FILES: - LOGGER.info("Checking file: " + lease) + LOGGER.info('Checking file: ' + lease) if os.path.exists(lease): - LOGGER.info("File Exists: " + lease) + LOGGER.info('File Exists: ' + lease) try: # Delete existing lease file - with (open(lease,'r')) as f: + with (open(lease, 'r', encoding='UTF-8')) as f: contents = f.read() while ip_addr in contents: - ixIp = contents.find(ip_addr) - leaseStart = contents.rindex("lease",0,ixIp) - leaseEnd = contents.find("}",leaseStart) - LOGGER.info("Lease Location: " + str(leaseStart)+":"+str(leaseEnd)) - contents = contents[0:leaseStart] + contents[leaseEnd+1:] + ix_ip = contents.find(ip_addr) + lease_start = contents.rindex('lease', 0, ix_ip) + lease_end = contents.find('}', lease_start) + LOGGER.info('Lease Location: ' + str(lease_start) + ':' + + str(lease_end)) + contents = contents[0:lease_start] + contents[lease_end + 1:] except OSError as e: - LOGGER.info(f"Error occurred while deleting the lease: {e}") - + LOGGER.info(f'Error occurred while deleting the lease: {e}') def _get_lease_list(self): LOGGER.info('Running lease list command') try: - stdout, stderr = util.run_command('dhcp-lease-list') - return stdout - except Exception as e: - LOGGER.error("Error lease list: " + str(e)) - - def _write_config(self,config): - with open(DHCP_CONFIG_FILE,"w") as f: - f.write(config) \ No newline at end of file + result = util.run_command('dhcp-lease-list') + return result[0] + except Exception as e: # pylint: disable=W0718 + LOGGER.error('Error lease list: ' + str(e)) + + def _write_config(self, config): + with open(DHCP_CONFIG_FILE, 'w', encoding='UTF-8') as f: + f.write(config) diff --git a/modules/network/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc/network_service.py index 81ede3cb0..bf2b98803 100644 --- a/modules/network/dhcp-1/python/src/grpc/network_service.py +++ b/modules/network/dhcp-1/python/src/grpc/network_service.py @@ -19,7 +19,10 @@ from dhcp_leases import DHCPLeases import traceback +from common import logger +LOG_NAME = 'network_service' +LOGGER = None class NetworkService(pb2_grpc.NetworkModule): """gRPC endpoints for the DHCP Server""" @@ -27,6 +30,8 @@ class NetworkService(pb2_grpc.NetworkModule): def __init__(self): self._dhcp_config = None self.dhcp_leases = DHCPLeases() + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') def _get_dhcp_config(self): if self._dhcp_config is None: @@ -35,85 +40,112 @@ def _get_dhcp_config(self): return self._dhcp_config def AddReservedLease(self, request, context): # pylint: disable=W0613 - print("Add Reserved Lease Called") + LOGGER.info('Add reserved lease called') try: dhcp_config = self._get_dhcp_config() - dhcp_config.add_reserved_host(request.hostname,request.hw_addr,request.ip_addr) + dhcp_config.add_reserved_host(request.hostname, request.hw_addr, + request.ip_addr) dhcp_config.write_config() - print("Reserve Leased Added") - except Exception as e: - print("Failed: " + str(e)) - traceback.print_exc() - return pb2.Response(code=200, message='{}') + LOGGER.info('Reserved lease added') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to add reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) def DeleteReservedLease(self, request, context): # pylint: disable=W0613 - print("Delete Reserved Lease Called") + LOGGER.info('Delete reserved lease called') try: dhcp_config = self._get_dhcp_config() dhcp_config.delete_reserved_host(request.hw_addr) dhcp_config.write_config() - print("Reserve Leased Deleted") - except Exception as e: - print("Failed: " + str(e)) - traceback.print_exc() - return pb2.Response(code=200, message='{}') - - def DisableFailover(self, request, contest): # pylint: disable=W0613 - print("Disabling Failover") + LOGGER.info('Reserved lease deleted') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to delete reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DisableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Disable failover called') try: dhcp_config = self._get_dhcp_config() dhcp_config.disable_failover() dhcp_config.write_config() - print("Failover Disabled") - except Exception as e: - print("Failed: " + str(e)) - return pb2.Response(code=200, message='{}') - - def EnableFailover(self, request, contest): # pylint: disable=W0613 - print("Enable Failover") + LOGGER.info('Failover disabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to disable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def EnableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Enable failover called') try: dhcp_config = self._get_dhcp_config() dhcp_config.enable_failover() dhcp_config.write_config() - print("Failover Enabled") - except Exception as e: - print("Failed: " + str(e)) - return pb2.Response(code=200, message='{}') + LOGGER.info('Failover enabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to enable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) def GetDHCPRange(self, request, context): # pylint: disable=W0613 """ Resolve the current DHCP configuration and return the first range from the first subnet in the file """ - pool = self._get_dhcp_config()._subnets[0].pools[0] - return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + LOGGER.info('Get DHCP range called') + try: + pool = self._get_dhcp_config()._subnets[0].pools[0] + return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) def GetLease(self, request, context): # pylint: disable=W0613 """ Resolve the current DHCP leased address for the provided MAC address """ - lease = self.dhcp_leases.get_lease(request.hw_addr) - if lease is not None: - return pb2.Response(code=200, message=str(lease)) - else: - return pb2.Response(code=200, message='{}') - + LOGGER.info('Get lease called') + try: + lease = self.dhcp_leases.get_lease(request.hw_addr) + if lease is not None: + return pb2.Response(code=200, message=str(lease)) + else: + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) def SetDHCPRange(self, request, context): # pylint: disable=W0613 """ Change DHCP configuration and set the the first range from the first subnet in the configuration """ - + LOGGER.info('Set DHCP range called') try: dhcp_config = self._get_dhcp_config() dhcp_config.set_range(request.start, request.end, 0, 0) dhcp_config.write_config() + LOGGER.info('DHCP range set') return pb2.Response(code=200, message='DHCP Range Set') - except Exception as e: - print("Failed: " + str(e)) - return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to set DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) def GetStatus(self, request, context): # pylint: disable=W0613 """ diff --git a/run_tests.sh b/run_tests.sh index 017d5f6a8..ab1656b58 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e - -export PYTHONPATH="$PWD/framework/python/src" -python3 -u modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py +#!/bin/bash -e + +export PYTHONPATH="$PWD/framework/python/src" +python3 -u modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py From c53263b30f5d6e3e4f881f2c252d937b762f531e Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 30 Jun 2023 16:20:37 -0600 Subject: [PATCH 14/25] Add python path resolving to network modules Update grpc path to prevent conflicts misc pylinting --- modules/network/base/bin/setup_python_path | 25 +++++++++++++++++++ modules/network/base/bin/start_grpc | 6 ++--- modules/network/base/bin/start_module | 12 ++++++--- .../src/{grpc => grpc_server}/start_server.py | 0 .../src/{grpc => grpc_server}/__init__.py | 0 .../src/{grpc => grpc_server}/dhcp_config.py | 18 +++---------- .../{grpc => grpc_server}/dhcp_config_test.py | 0 .../src/{grpc => grpc_server}/dhcp_lease.py | 0 .../src/{grpc => grpc_server}/dhcp_leases.py | 9 +------ .../{grpc => grpc_server}/network_service.py | 0 .../{grpc => grpc_server}/proto/grpc.proto | 0 .../src/{grpc => grpc_server}/__init__.py | 0 .../src/{grpc => grpc_server}/dhcp_config.py | 0 .../{grpc => grpc_server}/network_service.py | 0 .../{grpc => grpc_server}/proto/grpc.proto | 0 modules/test/base/base.Dockerfile | 4 +-- modules/test/base/bin/setup_grpc_clients | 2 +- modules/test/base/bin/start_module | 6 ++--- 18 files changed, 48 insertions(+), 34 deletions(-) create mode 100644 modules/network/base/bin/setup_python_path rename modules/network/base/python/src/{grpc => grpc_server}/start_server.py (100%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/__init__.py (100%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/dhcp_config.py (94%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/dhcp_config_test.py (100%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/dhcp_lease.py (100%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/dhcp_leases.py (91%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/network_service.py (100%) rename modules/network/dhcp-1/python/src/{grpc => grpc_server}/proto/grpc.proto (100%) rename modules/network/dhcp-2/python/src/{grpc => grpc_server}/__init__.py (100%) rename modules/network/dhcp-2/python/src/{grpc => grpc_server}/dhcp_config.py (100%) rename modules/network/dhcp-2/python/src/{grpc => grpc_server}/network_service.py (100%) rename modules/network/dhcp-2/python/src/{grpc => grpc_server}/proto/grpc.proto (100%) diff --git a/modules/network/base/bin/setup_python_path b/modules/network/base/bin/setup_python_path new file mode 100644 index 000000000..3e30e965d --- /dev/null +++ b/modules/network/base/bin/setup_python_path @@ -0,0 +1,25 @@ +#!/bin/bash + +ROOT_DIRECTORY="/testrun/python/src" + +# Function to recursively add subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath() { + local directory=$1 + local subdirectories=( "$directory"/* ) + local subdirectory + + for subdirectory in "${subdirectories[@]}"; do + if [[ -d "$subdirectory" && ! "$subdirectory" = *'__pycache__' ]]; then + export PYTHONPATH="$PYTHONPATH:$subdirectory" + add_subdirectories_to_pythonpath "$subdirectory" + fi + done +} + +# Set PYTHONPATH initially to an empty string +export PYTHONPATH="$ROOT_DIRECTORY" + +# Add all subdirectories to PYTHONPATH +add_subdirectories_to_pythonpath "$ROOT_DIRECTORY" + +echo "$PYTHONPATH" \ No newline at end of file diff --git a/modules/network/base/bin/start_grpc b/modules/network/base/bin/start_grpc index 9792b4bd4..38c261fc0 100644 --- a/modules/network/base/bin/start_grpc +++ b/modules/network/base/bin/start_grpc @@ -1,6 +1,6 @@ #!/bin/bash -e -GRPC_DIR="/testrun/python/src/grpc" +GRPC_DIR="/testrun/python/src/grpc_server" GRPC_PROTO_DIR="proto" GRPC_PROTO_FILE="grpc.proto" @@ -8,10 +8,10 @@ GRPC_PROTO_FILE="grpc.proto" pushd $GRPC_DIR >/dev/null 2>&1 #Build the grpc proto file every time before starting server -python3 -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. +python3 -u -m grpc_tools.protoc --proto_path=. ./$GRPC_PROTO_DIR/$GRPC_PROTO_FILE --python_out=. --grpc_python_out=. popd >/dev/null 2>&1 #Start the grpc server -python3 -u $GRPC_DIR/start_server.py $@ +python3 -u $GRPC_DIR/start_server.py $@ & diff --git a/modules/network/base/bin/start_module b/modules/network/base/bin/start_module index 7fdcbc404..ef83b9b73 100644 --- a/modules/network/base/bin/start_module +++ b/modules/network/base/bin/start_module @@ -46,10 +46,16 @@ else INTF=$DEFINED_IFACE fi -echo "Starting module $MODULE_NAME on local interface $INTF..." +# Setup the PYTHONPATH so all imports work as expected +echo "Setting up PYTHONPATH..." +export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" +echo "Configuring binary files..." $BIN_DIR/setup_binaries $BIN_DIR +echo "Starting module $MODULE_NAME on local interface $INTF..." + # Wait for interface to become ready $BIN_DIR/wait_for_interface $INTF @@ -66,9 +72,9 @@ then if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] then echo "gRPC port resolved from config: $GRPC_PORT" - $BIN_DIR/start_grpc "-p $GRPC_PORT" & + $BIN_DIR/start_grpc "-p $GRPC_PORT" else - $BIN_DIR/start_grpc & + $BIN_DIR/start_grpc fi fi diff --git a/modules/network/base/python/src/grpc/start_server.py b/modules/network/base/python/src/grpc_server/start_server.py similarity index 100% rename from modules/network/base/python/src/grpc/start_server.py rename to modules/network/base/python/src/grpc_server/start_server.py diff --git a/modules/network/dhcp-1/python/src/grpc/__init__.py b/modules/network/dhcp-1/python/src/grpc_server/__init__.py similarity index 100% rename from modules/network/dhcp-1/python/src/grpc/__init__.py rename to modules/network/dhcp-1/python/src/grpc_server/__init__.py diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py similarity index 94% rename from modules/network/dhcp-1/python/src/grpc/dhcp_config.py rename to modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py index 83f3cbe5a..111c33508 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_config.py +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py @@ -14,26 +14,16 @@ """Contains all the necessary classes to maintain the DHCP server's configuration""" import re -import os -import sys - - - -# Add the parent directory to sys.path -current_dir = os.path.dirname(os.path.abspath(__file__)) -parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) - -import common.logger as logger +from common import logger LOG_NAME = 'dhcp_config' LOGGER = None CONFIG_FILE = '/etc/dhcp/dhcpd.conf' -CONFIG_FILE_TEST = 'network/modules/dhcp-1/conf/dhcpd.conf' DEFAULT_LEASE_TIME_KEY = 'default-lease-time' + class DHCPConfig: """Represents the DHCP Servers configuration and gives access to modify it""" @@ -91,7 +81,7 @@ def make(self, conf): self._subnets = self.resolve_subnets(conf) self._peer = DHCPFailoverPeer(conf) self._reserved_hosts = self.resolve_reserved_hosts(conf) - except Exception as e: # pylint: disable=W0718 + except Exception as e: # pylint: disable=W0718 print('Failed to make DHCPConfig: ' + str(e)) def resolve_config(self, config_file=CONFIG_FILE): @@ -100,7 +90,7 @@ def resolve_config(self, config_file=CONFIG_FILE): self._subnets = self.resolve_subnets(conf) self._peer = DHCPFailoverPeer(conf) self._reserved_hosts = self.resolve_reserved_hosts(conf) - except Exception as e: # pylint: disable=W0718 + except Exception as e: # pylint: disable=W0718 print('Failed to resolve config: ' + str(e)) def resolve_subnets(self, conf): diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py similarity index 100% rename from modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py rename to modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_lease.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_lease.py similarity index 100% rename from modules/network/dhcp-1/python/src/grpc/dhcp_lease.py rename to modules/network/dhcp-1/python/src/grpc_server/dhcp_lease.py diff --git a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py similarity index 91% rename from modules/network/dhcp-1/python/src/grpc/dhcp_leases.py rename to modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py index fbfeb7b6c..63791bade 100644 --- a/modules/network/dhcp-1/python/src/grpc/dhcp_leases.py +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py @@ -13,14 +13,7 @@ # limitations under the License. """Used to resolve the DHCP servers lease information""" import os -import sys from dhcp_lease import DHCPLease - -# Add the parent directory to sys.path -current_dir = os.path.dirname(os.path.abspath(__file__)) -parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) - import logger from common import util @@ -105,7 +98,7 @@ def _get_lease_list(self): try: result = util.run_command('dhcp-lease-list') return result[0] - except Exception as e: # pylint: disable=W0718 + except Exception as e: # pylint: disable=W0718 LOGGER.error('Error lease list: ' + str(e)) def _write_config(self, config): diff --git a/modules/network/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc_server/network_service.py similarity index 100% rename from modules/network/dhcp-1/python/src/grpc/network_service.py rename to modules/network/dhcp-1/python/src/grpc_server/network_service.py diff --git a/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto similarity index 100% rename from modules/network/dhcp-1/python/src/grpc/proto/grpc.proto rename to modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto diff --git a/modules/network/dhcp-2/python/src/grpc/__init__.py b/modules/network/dhcp-2/python/src/grpc_server/__init__.py similarity index 100% rename from modules/network/dhcp-2/python/src/grpc/__init__.py rename to modules/network/dhcp-2/python/src/grpc_server/__init__.py diff --git a/modules/network/dhcp-2/python/src/grpc/dhcp_config.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py similarity index 100% rename from modules/network/dhcp-2/python/src/grpc/dhcp_config.py rename to modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py diff --git a/modules/network/dhcp-2/python/src/grpc/network_service.py b/modules/network/dhcp-2/python/src/grpc_server/network_service.py similarity index 100% rename from modules/network/dhcp-2/python/src/grpc/network_service.py rename to modules/network/dhcp-2/python/src/grpc_server/network_service.py diff --git a/modules/network/dhcp-2/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto similarity index 100% rename from modules/network/dhcp-2/python/src/grpc/proto/grpc.proto rename to modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index e12c34b36..d6a1bf9b3 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -24,8 +24,8 @@ RUN chmod u+x /testrun/bin/* # Copy over all network module gRPC proto files ARG NET_MODULE_DIR=modules/network -ARG NET_MODULE_PROTO_DIR=python/src/grpc/proto/grpc.proto -ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc/proto +ARG NET_MODULE_PROTO_DIR=python/src/grpc_server/proto/grpc.proto +ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc_server/proto COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ diff --git a/modules/test/base/bin/setup_grpc_clients b/modules/test/base/bin/setup_grpc_clients index 8cc003cfa..30efe5002 100644 --- a/modules/test/base/bin/setup_grpc_clients +++ b/modules/test/base/bin/setup_grpc_clients @@ -1,6 +1,6 @@ #!/bin/bash -e -GRPC_DIR="/testrun/python/src/grpc" +GRPC_DIR="/testrun/python/src/grpc_server" GRPC_PROTO_DIR="proto" GRPC_PROTO_FILE="grpc.proto" diff --git a/modules/test/base/bin/start_module b/modules/test/base/bin/start_module index fff724323..9e06cf005 100644 --- a/modules/test/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -45,8 +45,8 @@ fi echo "Setting up PYTHONPATH..." # Setup the PYTHONPATH so all imports work as expected -export PYTHONPATH=$($BIN_DIR/setup_python_path) echo "PYTHONPATH: $PYTHONPATH" +export PYTHONPATH=$($BIN_DIR/setup_python_path) # Build all gRPC files from the proto for use in # gRPC clients for communications to network modules @@ -75,9 +75,9 @@ then if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] then echo "gRPC port resolved from config: $GRPC_PORT" - $BIN_DIR/start_grpc "-p $GRPC_PORT" & + $BIN_DIR/start_grpc "-p $GRPC_PORT" else - $BIN_DIR/start_grpc & + $BIN_DIR/start_grpc fi fi From 62d46b7013c0cbbc7e89ab0947aea2263856f806 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 30 Jun 2023 16:56:42 -0600 Subject: [PATCH 15/25] Change lease resolving method to fix pylint issue --- .../network/dhcp-1/python/src/grpc_server/dhcp_leases.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py index 63791bade..698277a02 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_leases.py @@ -58,9 +58,10 @@ def get_leases(self): leases = [] lease_list_raw = self._get_lease_list() LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + '\n') - lines = lease_list_raw.split( - '===============================================================================================' - )[1].split('\n') + lease_list_start = lease_list_raw.find('=========',0) + lease_list_start = lease_list_raw.find('\n',lease_list_start) + lease_list = lease_list_raw[lease_list_start+1:] + lines = lease_list.split('\n') for line in lines: try: lease = DHCPLease(line) From 227ad5816271a9a932aa90de768e973c1a7bf266 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 3 Jul 2023 10:34:27 -0600 Subject: [PATCH 16/25] cleanup unit tests --- testing/unit_test/run_tests.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 testing/unit_test/run_tests.sh diff --git a/testing/unit_test/run_tests.sh b/testing/unit_test/run_tests.sh new file mode 100644 index 000000000..ea2dbdef5 --- /dev/null +++ b/testing/unit_test/run_tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +# This script should be run from within the unit_test directory. If +# it is run outside this directory, paths will not be resolved correctly. + +# Move into the root directory of test-run +pushd ../../ >/dev/null 2>&1 + +echo "Root Dir: $PWD" + +# Setup the python path +export PYTHONPATH="$PWD/framework/python/src" + +# Run the DHCP Unit tests +python3 -u $PWD/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py + +popd >/dev/null 2>&1 \ No newline at end of file From 976f1164ea23c7bf10f019f915d24b769393f7c2 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 3 Jul 2023 10:37:59 -0600 Subject: [PATCH 17/25] cleanup unit tests --- .../python/src/grpc_server/dhcp_config_test.py | 17 +++++++++++++++-- run_tests.sh | 4 ---- 2 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 run_tests.sh diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py index 66b2c42e6..2cc78403a 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit Testing for the DHCP Server config""" import unittest from dhcp_config import DHCPConfig import os @@ -86,5 +100,4 @@ def test_resolve_config_with_hosts(self): suite.addTest(DHCPConfigTest('test_resolve_config_with_hosts')) runner = unittest.TextTestRunner() - runner.run(suite) - # unittest.main() \ No newline at end of file + runner.run(suite) \ No newline at end of file diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100644 index ab1656b58..000000000 --- a/run_tests.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -e - -export PYTHONPATH="$PWD/framework/python/src" -python3 -u modules/network/dhcp-1/python/src/grpc/dhcp_config_test.py From 8a2526bc38a589ba6e0f90e44ee2989f42d12927 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 3 Jul 2023 12:07:18 -0600 Subject: [PATCH 18/25] Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables --- testing/unit_test/run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/unit_test/run_tests.sh b/testing/unit_test/run_tests.sh index ea2dbdef5..5b1ed6257 100644 --- a/testing/unit_test/run_tests.sh +++ b/testing/unit_test/run_tests.sh @@ -13,5 +13,6 @@ export PYTHONPATH="$PWD/framework/python/src" # Run the DHCP Unit tests python3 -u $PWD/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py +python3 -u $PWD/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py popd >/dev/null 2>&1 \ No newline at end of file From 0fd5a8d0799181f4177955a0729caed32fd2b410 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 3 Jul 2023 12:07:52 -0600 Subject: [PATCH 19/25] Add grpc updates to dhcp2 module Update dhcp_config to deal with missing optional variables --- .../network/dhcp-1/bin/start_network_service | 3 - .../python/src/grpc_server/dhcp_config.py | 56 ++-- .../python/src/grpc_server/proto/grpc.proto | 11 - modules/network/dhcp-2/conf/dhcpd.conf | 35 +- .../python/src/grpc_server/dhcp_config.py | 312 ++++++++++++++---- .../src/grpc_server/dhcp_config_test.py | 103 ++++++ .../python/src/grpc_server/dhcp_lease.py | 75 +++++ .../python/src/grpc_server/dhcp_leases.py | 107 ++++++ .../python/src/grpc_server/network_service.py | 131 +++++++- .../python/src/grpc_server/proto/grpc.proto | 51 ++- 10 files changed, 735 insertions(+), 149 deletions(-) create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py diff --git a/modules/network/dhcp-1/bin/start_network_service b/modules/network/dhcp-1/bin/start_network_service index 1d6a26d57..4221a73f7 100644 --- a/modules/network/dhcp-1/bin/start_network_service +++ b/modules/network/dhcp-1/bin/start_network_service @@ -15,9 +15,6 @@ sysctl -p # Create leases file if needed touch /var/lib/dhcp/dhcpd.leases -# Create log file if needed -touch /var/log/dhcp/dhcpd.log - #Create directory for radvd mkdir /var/run/radvd diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py index 111c33508..444faa87c 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_config.py @@ -184,16 +184,16 @@ def __init__(self, config): def __str__(self): config = '{FAILOVER_PEER_KEY} \"{FAILOVER_PEER}\" {{\n' config += '\tprimary;' if self.primary else 'secondary;' - config += """\n\t{ADDRESS_KEY} {ADDRESS}; - {PORT_KEY} {PORT}; - {PEER_ADDRESS_KEY} {PEER_ADDRESS}; - {PEER_PORT_KEY} {PEER_PORT}; - {MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY}; - {MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES}; - {MCLT_KEY} {MCLT}; - {SPLIT_KEY} {SPLIT}; - {LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS}; - \r}}""" + config += '\n\t{ADDRESS_KEY} {ADDRESS};' if self.address is not None else '' + config += '\n\t{PORT_KEY} {PORT};' if self.port is not None else '' + config += '\n\t{PEER_ADDRESS_KEY} {PEER_ADDRESS};' if self.peer_address is not None else '' + config += '\n\t{PEER_PORT_KEY} {PEER_PORT};' if self.peer_port is not None else '' + config += '\n\t{MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY};' if self.max_response_delay is not None else '' + config += '\n\t{MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES};' if self.max_unacked_updates is not None else '' + config += '\n\t{MCLT_KEY} {MCLT};' if self.mclt is not None else '' + config += '\n\t{SPLIT_KEY} {SPLIT};' if self.split is not None else '' + config += '\n\t{LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS};' if self.load_balance_max_seconds is not None else '' + config += '\n\r}}' config = config.format( length='multi-line', @@ -220,8 +220,9 @@ def __str__(self): if not self.enabled: lines = config.strip().split('\n') - for i in range(len(lines)): + for i in range(len(lines)-1): lines[i] = '#' + lines[i] + lines[-1] = '#' + lines[-1].strip() # Handle the last line separately config = '\n'.join(lines) return config @@ -300,14 +301,15 @@ def __init__(self, subnet): self.resolve_pools(subnet) def __str__(self): - config = """subnet {SUBNET_OPTION} netmask {SUBNET_MASK_OPTION} {{ - \r\t{NTP_OPTION_KEY} {NTP_OPTION}; - \r\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION}; - \r\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION}; - \r\t{ROUTER_OPTION_KEY} {ROUTER_OPTION}; - \r\t{DNS_OPTION_KEY} {DNS_OPTION}; - \r\t{INTERFACE_KEY} {INTERFACE_OPTION}; - \r\tauthoritative;""" + config = 'subnet {SUBNET_OPTION} netmask {SUBNET_MASK_OPTION} {{' + config += '\n\t{NTP_OPTION_KEY} {NTP_OPTION};' if self._ntp_servers is not None else '' + config += '\n\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION};' if self._subnet_mask is not None else '' + config += '\n\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION};' if self._broadcast is not None else '' + config += '\n\t{ROUTER_OPTION_KEY} {ROUTER_OPTION};' if self._routers is not None else '' + config += '\n\t{DNS_OPTION_KEY} {DNS_OPTION};' if self._dns_servers is not None else '' + config += '\n\t{INTERFACE_KEY} {INTERFACE_OPTION};' if self._interface is not None else '' + config += '\n\t{AUTHORITATIVE_KEY};' if self._authoritative else '' + config = config.format(length='multi-line', SUBNET_OPTION=self._subnet, @@ -322,10 +324,11 @@ def __str__(self): DNS_OPTION_KEY=DNS_OPTION_KEY, DNS_OPTION=self._dns_servers, INTERFACE_KEY=INTERFACE_KEY, - INTERFACE_OPTION=self._interface) + INTERFACE_OPTION=self._interface, + AUTHORITATIVE_KEY=AUTHORITATIVE_KEY) - if not self._authoritative: - config = config.replace(AUTHORITATIVE_KEY, '#' + AUTHORITATIVE_KEY) + # if not self._authoritative: + # config = config.replace(AUTHORITATIVE_KEY, '#' + AUTHORITATIVE_KEY) for pool in self.pools: config += '\n\t' + str(pool) @@ -403,11 +406,10 @@ def __init__(self, pool): self._peer_enabled = True def __str__(self): - - config = """pool {{ - \r\t\t{FAILOVER_KEY} "{FAILOVER}"; - \r\t\t{RANGE_KEY} {RANGE_START} {RANGE_END}; - \r\t}}""" + config = 'pool {{' + config += '\n\t\t{FAILOVER_KEY} "{FAILOVER}";' if self.failover_peer is not None else '' + config += '\n\t\t{RANGE_KEY} {RANGE_START} {RANGE_END};' if self.range_start is not None and self.range_end is not None else '' + config += '\n\t}}' config = config.format( length='multi-line', diff --git a/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto index b5af4e9ec..d9f56213e 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto +++ b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto @@ -57,14 +57,3 @@ message DHCPRange { string start = 2; string end = 3; } - - - - - - - - - - - diff --git a/modules/network/dhcp-2/conf/dhcpd.conf b/modules/network/dhcp-2/conf/dhcpd.conf index e73a81441..dcc47a4fe 100644 --- a/modules/network/dhcp-2/conf/dhcpd.conf +++ b/modules/network/dhcp-2/conf/dhcpd.conf @@ -1,24 +1,25 @@ default-lease-time 300; failover peer "failover-peer" { - secondary; - address 10.10.10.3; - port 647; - peer address 10.10.10.2; - peer port 847; - max-response-delay 60; - max-unacked-updates 10; - load balance max seconds 3; + secondary; + address 10.10.10.3; + port 647; + peer address 10.10.10.2; + peer port 847; + max-response-delay 60; + max-unacked-updates 10; + load balance max seconds 3; } subnet 10.10.10.0 netmask 255.255.255.0 { - option ntp-servers 10.10.10.5; - option subnet-mask 255.255.255.0; - option broadcast-address 10.10.10.255; - option routers 10.10.10.1; - option domain-name-servers 10.10.10.4; - pool { - failover peer "failover-peer"; - range 10.10.10.10 10.10.10.20; - } + option ntp-servers 10.10.10.5; + option subnet-mask 255.255.255.0; + option broadcast-address 10.10.10.255; + option routers 10.10.10.1; + option domain-name-servers 10.10.10.4; + interface veth0; + pool { + failover peer "failover-peer"; + range 10.10.10.10 10.10.10.20; + } } diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py index f6e79a2ec..444faa87c 100644 --- a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py @@ -11,13 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Contains all the necessary classes to maintain the DHCP server's configuration""" import re +from common import logger + +LOG_NAME = 'dhcp_config' +LOGGER = None CONFIG_FILE = '/etc/dhcp/dhcpd.conf' -CONFIG_FILE_TEST = 'network/modules/dhcp-2/conf/dhcpd.conf' DEFAULT_LEASE_TIME_KEY = 'default-lease-time' @@ -27,55 +29,122 @@ class DHCPConfig: def __init__(self): self._default_lease_time = 300 - self.subnets = [] + self._subnets = [] self._peer = None + self._reserved_hosts = [] + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def add_reserved_host(self, hostname, hw_addr, ip_addr): + host = DHCPReservedHost(hostname=hostname, + hw_addr=hw_addr, + fixed_addr=ip_addr) + self._reserved_hosts.append(host) + + def delete_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host.hw_addr: + self._reserved_hosts.remove(host) + + def disable_failover(self): + self._peer.disable() + for subnet in self._subnets: + subnet.disable_peer() - def write_config(self): - conf = str(self) - print('Writing config: \n' + conf) - with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: - conf_file.write(conf) - - def resolve_config(self): - with open(CONFIG_FILE, 'r', encoding='UTF-8') as f: - conf = f.read() - self.resolve_subnets(conf) - self._peer = DHCPFailoverPeer(conf) + def enable_failover(self): + self._peer.enable() + for subnet in self._subnets: + subnet.enable_peer() + + def get_reserved_host(self, hw_addr): + for host in self._reserved_hosts: + if hw_addr == host.hw_addr: + return host + + def write_config(self, config=None): + if config is None: + conf = str(self) + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(conf) + else: + with open(CONFIG_FILE, 'w', encoding='UTF-8') as conf_file: + conf_file.write(config) + + def _get_config(self, config_file=CONFIG_FILE): + content = None + with open(config_file, 'r', encoding='UTF-8') as f: + content = f.read() + return content + + def make(self, conf): + try: + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: # pylint: disable=W0718 + print('Failed to make DHCPConfig: ' + str(e)) + + def resolve_config(self, config_file=CONFIG_FILE): + try: + conf = self._get_config(config_file) + self._subnets = self.resolve_subnets(conf) + self._peer = DHCPFailoverPeer(conf) + self._reserved_hosts = self.resolve_reserved_hosts(conf) + except Exception as e: # pylint: disable=W0718 + print('Failed to resolve config: ' + str(e)) def resolve_subnets(self, conf): - self.subnets = [] + subnets = [] regex = r'(subnet.*)' - subnets = re.findall(regex, conf, re.MULTILINE | re.DOTALL) - for subnet in subnets: + subnets_conf = re.findall(regex, conf, re.MULTILINE | re.DOTALL) + for subnet in subnets_conf: dhcp_subnet = DHCPSubnet(subnet) - self.subnets.append(dhcp_subnet) + subnets.append(dhcp_subnet) + return subnets + + def resolve_reserved_hosts(self, conf): + hosts = [] + host_start = 0 + while True: + host_start = conf.find('host', host_start) + if host_start < 0: + break + else: + host_end = conf.find('}', host_start) + host = DHCPReservedHost(config=conf[host_start:host_end + 1]) + hosts.append(host) + host_start = host_end + 1 + return hosts def set_range(self, start, end, subnet=0, pool=0): - print('Setting Range for pool ') - print(self.subnets[subnet].pools[pool]) - self.subnets[subnet].pools[pool].range_start = start - self.subnets[subnet].pools[pool].range_end = end + # Calculate the subnet from the range + octets = start.split('.') + octets[-1] = '0' + dhcp_subnet = '.'.join(octets) - # def resolve_settings(self, conf): - # lines = conf.split('\n') - # for line in lines: - # if DEFAULT_LEASE_TIME_KEY in line: - # self._default_lease_time = line.strip().split( - # DEFAULT_LEASE_TIME_KEY)[1].strip().split(';')[0] - - # self.peer = peer + #Update the subnet and range + self._subnets[subnet].set_subnet(dhcp_subnet) + self._subnets[subnet].pools[pool].set_range(start, end) def __str__(self): - config = """\r{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" - + # Encode the top level config options + config = """{DEFAULT_LEASE_TIME_KEY} {DEFAULT_LEASE_TIME};""" config = config.format(length='multi-line', DEFAULT_LEASE_TIME_KEY=DEFAULT_LEASE_TIME_KEY, DEFAULT_LEASE_TIME=self._default_lease_time) - config += '\n\n' + str(self.peer) + # Encode the failover peer + config += '\n\n' + str(self._peer) + + # Encode the subnets for subnet in self._subnets: config += '\n\n' + str(subnet) + + # Encode the reserved hosts + for host in self._reserved_hosts: + config += '\n' + str(host) + return str(config) @@ -108,24 +177,25 @@ def __init__(self, config): self.split = None self.load_balance_max_seconds = None self.peer = None + self.enabled = True self.resolve_peer(config) def __str__(self): config = '{FAILOVER_PEER_KEY} \"{FAILOVER_PEER}\" {{\n' config += '\tprimary;' if self.primary else 'secondary;' - config += """\n\t{ADDRESS_KEY} {ADDRESS}; - {PORT_KEY} {PORT}; - {PEER_ADDRESS_KEY} {PEER_ADDRESS}; - {PEER_PORT_KEY} {PEER_PORT}; - {MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY}; - {MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES}; - {MCLT_KEY} {MCLT}; - {SPLIT_KEY} {SPLIT}; - {LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS}; - \r}}""" - - return config.format( + config += '\n\t{ADDRESS_KEY} {ADDRESS};' if self.address is not None else '' + config += '\n\t{PORT_KEY} {PORT};' if self.port is not None else '' + config += '\n\t{PEER_ADDRESS_KEY} {PEER_ADDRESS};' if self.peer_address is not None else '' + config += '\n\t{PEER_PORT_KEY} {PEER_PORT};' if self.peer_port is not None else '' + config += '\n\t{MAX_RESPONSE_DELAY_KEY} {MAX_RESPONSE_DELAY};' if self.max_response_delay is not None else '' + config += '\n\t{MAX_UNACKED_UPDATES_KEY} {MAX_UNACKED_UPDATES};' if self.max_unacked_updates is not None else '' + config += '\n\t{MCLT_KEY} {MCLT};' if self.mclt is not None else '' + config += '\n\t{SPLIT_KEY} {SPLIT};' if self.split is not None else '' + config += '\n\t{LOAD_BALANCE_MAX_SECONDS_KEY} {LOAD_BALANCE_MAX_SECONDS};' if self.load_balance_max_seconds is not None else '' + config += '\n\r}}' + + config = config.format( length='multi-line', FAILOVER_PEER_KEY=FAILOVER_PEER_KEY, FAILOVER_PEER=self.name, @@ -148,6 +218,21 @@ def __str__(self): LOAD_BALANCE_MAX_SECONDS_KEY=LOAD_BALANCE_MAX_SECONDS_KEY, LOAD_BALANCE_MAX_SECONDS=self.load_balance_max_seconds) + if not self.enabled: + lines = config.strip().split('\n') + for i in range(len(lines)-1): + lines[i] = '#' + lines[i] + lines[-1] = '#' + lines[-1].strip() # Handle the last line separately + config = '\n'.join(lines) + + return config + + def disable(self): + self.enabled = False + + def enable(self): + self.enabled = True + def resolve_peer(self, conf): peer = '' lines = conf.split('\n') @@ -188,36 +273,46 @@ def resolve_peer(self, conf): self.peer = peer +SUBNET_KEY = 'subnet' NTP_OPTION_KEY = 'option ntp-servers' SUBNET_MASK_OPTION_KEY = 'option subnet-mask' BROADCAST_OPTION_KEY = 'option broadcast-address' ROUTER_OPTION_KEY = 'option routers' DNS_OPTION_KEY = 'option domain-name-servers' +INTERFACE_KEY = 'interface' +AUTHORITATIVE_KEY = 'authoritative' class DHCPSubnet: """Represents the DHCP Servers subnet configuration""" def __init__(self, subnet): + self._authoritative = False + self._subnet = None self._ntp_servers = None self._subnet_mask = None self._broadcast = None self._routers = None self._dns_servers = None + self._interface = None self.pools = [] self.resolve_subnet(subnet) self.resolve_pools(subnet) def __str__(self): - config = """subnet 10.10.10.0 netmask {SUBNET_MASK_OPTION} {{ - \r\t{NTP_OPTION_KEY} {NTP_OPTION}; - \r\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION}; - \r\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION}; - \r\t{ROUTER_OPTION_KEY} {ROUTER_OPTION}; - \r\t{DNS_OPTION_KEY} {DNS_OPTION};""" + config = 'subnet {SUBNET_OPTION} netmask {SUBNET_MASK_OPTION} {{' + config += '\n\t{NTP_OPTION_KEY} {NTP_OPTION};' if self._ntp_servers is not None else '' + config += '\n\t{SUBNET_MASK_OPTION_KEY} {SUBNET_MASK_OPTION};' if self._subnet_mask is not None else '' + config += '\n\t{BROADCAST_OPTION_KEY} {BROADCAST_OPTION};' if self._broadcast is not None else '' + config += '\n\t{ROUTER_OPTION_KEY} {ROUTER_OPTION};' if self._routers is not None else '' + config += '\n\t{DNS_OPTION_KEY} {DNS_OPTION};' if self._dns_servers is not None else '' + config += '\n\t{INTERFACE_KEY} {INTERFACE_OPTION};' if self._interface is not None else '' + config += '\n\t{AUTHORITATIVE_KEY};' if self._authoritative else '' + config = config.format(length='multi-line', + SUBNET_OPTION=self._subnet, NTP_OPTION_KEY=NTP_OPTION_KEY, NTP_OPTION=self._ntp_servers, SUBNET_MASK_OPTION_KEY=SUBNET_MASK_OPTION_KEY, @@ -227,17 +322,47 @@ def __str__(self): ROUTER_OPTION_KEY=ROUTER_OPTION_KEY, ROUTER_OPTION=self._routers, DNS_OPTION_KEY=DNS_OPTION_KEY, - DNS_OPTION=self._dns_servers) + DNS_OPTION=self._dns_servers, + INTERFACE_KEY=INTERFACE_KEY, + INTERFACE_OPTION=self._interface, + AUTHORITATIVE_KEY=AUTHORITATIVE_KEY) + + # if not self._authoritative: + # config = config.replace(AUTHORITATIVE_KEY, '#' + AUTHORITATIVE_KEY) + for pool in self.pools: config += '\n\t' + str(pool) - config += '\n\r}' + config += '\n}' return config + def disable_peer(self): + for pool in self.pools: + pool.disable_peer() + + def enable_peer(self): + for pool in self.pools: + pool.enable_peer() + + def set_subnet(self, subnet, netmask=None): + if netmask is None: + netmask = '255.255.255.0' + self._subnet = subnet + self._subnet_mask = netmask + + # Calculate the broadcast from the subnet + octets = subnet.split('.') + octets[-1] = '255' + dhcp_broadcast = '.'.join(octets) + + self._broadcast = dhcp_broadcast + def resolve_subnet(self, subnet): subnet_parts = subnet.split('\n') for part in subnet_parts: - if NTP_OPTION_KEY in part: + if part.strip().startswith(SUBNET_KEY): + self._subnet = part.strip().split()[1] + elif NTP_OPTION_KEY in part: self._ntp_servers = part.strip().split(NTP_OPTION_KEY)[1].strip().split( ';')[0] elif SUBNET_MASK_OPTION_KEY in part: @@ -252,6 +377,11 @@ def resolve_subnet(self, subnet): elif DNS_OPTION_KEY in part: self._dns_servers = part.strip().split(DNS_OPTION_KEY)[1].strip().split( ';')[0] + elif INTERFACE_KEY in part: + self._interface = part.strip().split(INTERFACE_KEY)[1].strip().split( + ';')[0] + elif AUTHORITATIVE_KEY in part: + self._authoritative = True def resolve_pools(self, subnet): regex = r'(pool.*)\}' @@ -273,15 +403,15 @@ def __init__(self, pool): self.range_start = None self.range_end = None self.resolve_pool(pool) + self._peer_enabled = True def __str__(self): + config = 'pool {{' + config += '\n\t\t{FAILOVER_KEY} "{FAILOVER}";' if self.failover_peer is not None else '' + config += '\n\t\t{RANGE_KEY} {RANGE_START} {RANGE_END};' if self.range_start is not None and self.range_end is not None else '' + config += '\n\t}}' - config = """pool {{ - \r\t\t{FAILOVER_KEY} "{FAILOVER}"; - \r\t\t{RANGE_KEY} {RANGE_START} {RANGE_END}; - \r\t}}""" - - return config.format( + config = config.format( length='multi-line', FAILOVER_KEY=FAILOVER_KEY, FAILOVER=self.failover_peer, @@ -290,9 +420,23 @@ def __str__(self): RANGE_END=self.range_end, ) + if not self._peer_enabled: + config = config.replace(FAILOVER_KEY, '#' + FAILOVER_KEY) + + return config + + def disable_peer(self): + self._peer_enabled = False + + def enable_peer(self): + self._peer_enabled = True + + def set_range(self, start, end): + self.range_start = start + self.range_end = end + def resolve_pool(self, pool): pool_parts = pool.split('\n') - # pool_parts = pool.split("\n") for part in pool_parts: if FAILOVER_KEY in part: self.failover_peer = part.strip().split(FAILOVER_KEY)[1].strip().split( @@ -301,3 +445,49 @@ def resolve_pool(self, pool): pool_range = part.strip().split(RANGE_KEY)[1].strip().split(';')[0] self.range_start = pool_range.split(' ')[0].strip() self.range_end = pool_range.split(' ')[1].strip() + + +HOST_KEY = 'host' +HARDWARE_KEY = 'hardware ethernet' +FIXED_ADDRESS_KEY = 'fixed-address' + + +class DHCPReservedHost: + """Represents a DHCP Servers subnet pool configuration""" + + def __init__(self, hostname=None, hw_addr=None, fixed_addr=None, config=None): + if config is None: + self.host = hostname + self.hw_addr = hw_addr + self.fixed_addr = fixed_addr + else: + self.resolve_host(config) + + def __str__(self): + + config = """{HOST_KEY} {HOSTNAME} {{ + \r\t{HARDWARE_KEY} {HW_ADDR}; + \r\t{FIXED_ADDRESS_KEY} {RESERVED_IP}; + \r}}""" + + config = config.format( + length='multi-line', + HOST_KEY=HOST_KEY, + HOSTNAME=self.host, + HARDWARE_KEY=HARDWARE_KEY, + HW_ADDR=self.hw_addr, + FIXED_ADDRESS_KEY=FIXED_ADDRESS_KEY, + RESERVED_IP=self.fixed_addr, + ) + return config + + def resolve_host(self, reserved_host): + host_parts = reserved_host.split('\n') + for part in host_parts: + if HOST_KEY in part: + self.host = part.strip().split(HOST_KEY)[1].strip().split('{')[0] + elif HARDWARE_KEY in part: + self.hw_addr = part.strip().split(HARDWARE_KEY)[1].strip().split(';')[0] + elif FIXED_ADDRESS_KEY in part: + self.fixed_addr = part.strip().split( + FIXED_ADDRESS_KEY)[1].strip().split(';')[0] diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py new file mode 100644 index 000000000..2cc78403a --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py @@ -0,0 +1,103 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit Testing for the DHCP Server config""" +import unittest +from dhcp_config import DHCPConfig +import os + +CONFIG_FILE = 'conf/dhcpd.conf' + +DHCP_CONFIG = None + +def get_config_file_path(): + dhcp_config = DHCPConfig() + current_dir = os.path.dirname(os.path.abspath(__file__)) + module_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(current_dir)))) + conf_file = os.path.join(module_dir,CONFIG_FILE) + return conf_file + +def get_config(): + dhcp_config = DHCPConfig() + dhcp_config.resolve_config(get_config_file_path()) + return dhcp_config + +class DHCPConfigTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + # Resolve the config + global DHCP_CONFIG + DHCP_CONFIG = get_config() + + def test_resolve_config(self): + print('Test Resolve Config:\n' + str(DHCP_CONFIG)) + + # Resolve the raw config file + with open(get_config_file_path(),'r') as f: + lines = f.readlines() + + # Get the resolved config as a + conf_parts = str(DHCP_CONFIG).split('\n') + + # dhcpd conf is not picky about spacing so we just + # need to check contents of each line for matching + # to make sure evertying matches + for i in range(len(lines)): + self.assertEqual(lines[i].strip(),conf_parts[i].strip()) + + def test_disable_failover(self): + DHCP_CONFIG.disable_failover() + print('Test Disable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertTrue(line.startswith('#')) + + def test_enable_failover(self): + DHCP_CONFIG.enable_failover() + print('Test Enable Config:\n' + str(DHCP_CONFIG)) + config_lines = str(DHCP_CONFIG._peer).split('\n') + for line in config_lines: + self.assertFalse(line.startswith('#')) + + def test_add_reserved_host(self): + DHCP_CONFIG.add_reserved_host('test','00:11:22:33:44:55','192.168.10.5') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print('AddHostConfig:\n' + str(DHCP_CONFIG)) + + def test_delete_reserved_host(self): + DHCP_CONFIG.delete_reserved_host('00:11:22:33:44:55') + host = DHCP_CONFIG.get_reserved_host('00:11:22:33:44:55') + self.assertIsNone(host) + print('DeleteHostConfig:\n' + str(DHCP_CONFIG)) + + def test_resolve_config_with_hosts(self): + DHCP_CONFIG.add_reserved_host('test','00:11:22:33:44:55','192.168.10.5') + config_with_hosts = DHCPConfig() + config_with_hosts.make(str(DHCP_CONFIG)) + host = config_with_hosts.get_reserved_host('00:11:22:33:44:55') + self.assertIsNotNone(host) + print("ResolveConfigWithHosts:\n" + str(config_with_hosts)) + +if __name__ == '__main__': + suite = unittest.TestSuite() + suite.addTest(DHCPConfigTest('test_resolve_config')) + suite.addTest(DHCPConfigTest('test_disable_failover')) + suite.addTest(DHCPConfigTest('test_enable_failover')) + suite.addTest(DHCPConfigTest('test_add_reserved_host')) + suite.addTest(DHCPConfigTest('test_delete_reserved_host')) + suite.addTest(DHCPConfigTest('test_resolve_config_with_hosts')) + + runner = unittest.TextTestRunner() + runner.run(suite) \ No newline at end of file diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py new file mode 100644 index 000000000..0d2f43e3b --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_lease.py @@ -0,0 +1,75 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary methods to create and monitor DHCP +leases on the server""" +from datetime import datetime +import time + +time_format = '%Y-%m-%d %H:%M:%S' + + +class DHCPLease(object): + """Represents a DHCP Server lease""" + hw_addr = None + ip = None + hostname = None + expires = None + + def __init__(self, lease): + self._make_lease(lease) + + def _make_lease(self, lease): + if lease is not None: + sections_raw = lease.split(' ') + sections = [] + for section in sections_raw: + if section.strip(): + sections.append(section) + self.hw_addr = sections[0] + self.ip = sections[1] + self.hostname = sections[2] + self.expires = sections[3] + '' '' + sections[4] + self.manufacturer = ' '.join(sections[5:]) + + def get_millis(self, timestamp): + dt_obj = datetime.strptime(timestamp, time_format) + millis = dt_obj.timestamp() * 1000 + return millis + + def get_expires_millis(self): + return self.get_millis(self.expires) + + def is_expired(self): + expires_millis = self.get_expires_millis() + cur_time = int(round(time.time()) * 1000) + return cur_time >= expires_millis + + def __str__(self): + lease = {} + if self.hw_addr is not None: + lease['hw_addr'] = self.hw_addr + + if self.ip is not None: + lease['ip'] = self.ip + + if self.hostname is not None: + lease['hostname'] = self.hostname + + if self.expires is not None: + lease['expires'] = self.expires + + if self.manufacturer is not None: + lease['manufacturer'] = self.manufacturer + + return str(lease) diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py new file mode 100644 index 000000000..08e6feabe --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_leases.py @@ -0,0 +1,107 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Used to resolve the DHCP servers lease information""" +import os +from dhcp_lease import DHCPLease +import logger +from common import util + +LOG_NAME = 'dhcp_lease' +LOGGER = None + +DHCP_LEASE_FILES = [ + '/var/lib/dhcp/dhcpd.leases', '/var/lib/dhcp/dhcpd.leases~', + '/var/lib/dhcp/dhcpd6.leases', '/var/lib/dhcp/dhcpd6.leases~' +] +DHCP_CONFIG_FILE = '/etc/dhcp/dhcpd.conf' + + +class DHCPLeases: + """Leases for the DHCP server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def delete_all_hosts(self): + LOGGER.info('Deleting hosts') + for lease in DHCP_LEASE_FILES: + LOGGER.info('Checking file: ' + lease) + if os.path.exists(lease): + LOGGER.info('File Exists: ' + lease) + try: + # Delete existing lease file + os.remove(lease) + except OSError as e: + LOGGER.info(f'Error occurred while deleting the file: {e}') + # Create an empty lease file + with open(lease, 'w', encoding='UTF-8'): + pass + + def get_lease(self, hw_addr): + for lease in self.get_leases(): + if lease.hw_addr == hw_addr: + return lease + + def get_leases(self): + leases = [] + lease_list_raw = self._get_lease_list() + LOGGER.info('Raw Leases:\n' + str(lease_list_raw) + '\n') + lease_list_start = lease_list_raw.find('=========',0) + lease_list_start = lease_list_raw.find('\n',lease_list_start) + lease_list = lease_list_raw[lease_list_start+1:] + lines = lease_list.split('\n') + for line in lines: + try: + lease = DHCPLease(line) + leases.append(lease) + except Exception as e: # pylint: disable=W0718 + # Let non lease lines file without extra checks + LOGGER.error('Making Lease Error: ' + str(e)) + LOGGER.error('Not a valid lease line: ' + line) + return leases + + def delete_lease(self, ip_addr): + LOGGER.info('Deleting lease') + for lease in DHCP_LEASE_FILES: + LOGGER.info('Checking file: ' + lease) + if os.path.exists(lease): + LOGGER.info('File Exists: ' + lease) + try: + # Delete existing lease file + with (open(lease, 'r', encoding='UTF-8')) as f: + contents = f.read() + + while ip_addr in contents: + ix_ip = contents.find(ip_addr) + lease_start = contents.rindex('lease', 0, ix_ip) + lease_end = contents.find('}', lease_start) + LOGGER.info('Lease Location: ' + str(lease_start) + ':' + + str(lease_end)) + contents = contents[0:lease_start] + contents[lease_end + 1:] + + except OSError as e: + LOGGER.info(f'Error occurred while deleting the lease: {e}') + + def _get_lease_list(self): + LOGGER.info('Running lease list command') + try: + result = util.run_command('dhcp-lease-list') + return result[0] + except Exception as e: # pylint: disable=W0718 + LOGGER.error('Error lease list: ' + str(e)) + + def _write_config(self, config): + with open(DHCP_CONFIG_FILE, 'w', encoding='UTF-8') as f: + f.write(config) diff --git a/modules/network/dhcp-2/python/src/grpc_server/network_service.py b/modules/network/dhcp-2/python/src/grpc_server/network_service.py index 64aab8a07..053d26d6b 100644 --- a/modules/network/dhcp-2/python/src/grpc_server/network_service.py +++ b/modules/network/dhcp-2/python/src/grpc_server/network_service.py @@ -11,44 +11,143 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """gRPC Network Service for the DHCP Server network module""" import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 from dhcp_config import DHCPConfig +from dhcp_leases import DHCPLeases + +import traceback +from common import logger +LOG_NAME = 'network_service' +LOGGER = None class NetworkService(pb2_grpc.NetworkModule): """gRPC endpoints for the DHCP Server""" def __init__(self): - self._dhcp_config = DHCPConfig() + self._dhcp_config = None + self.dhcp_leases = DHCPLeases() + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def _get_dhcp_config(self): + if self._dhcp_config is None: + self._dhcp_config = DHCPConfig() + self._dhcp_config.resolve_config() + return self._dhcp_config + + def AddReservedLease(self, request, context): # pylint: disable=W0613 + LOGGER.info('Add reserved lease called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.add_reserved_host(request.hostname, request.hw_addr, + request.ip_addr) + dhcp_config.write_config() + LOGGER.info('Reserved lease added') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to add reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DeleteReservedLease(self, request, context): # pylint: disable=W0613 + LOGGER.info('Delete reserved lease called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.delete_reserved_host(request.hw_addr) + dhcp_config.write_config() + LOGGER.info('Reserved lease deleted') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to delete reserved lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def DisableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Disable failover called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.disable_failover() + dhcp_config.write_config() + LOGGER.info('Failover disabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to disable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) - def GetDHCPRange(self, request, context): # pylint: disable=W0613 + def EnableFailover(self, request, contest): # pylint: disable=W0613 + LOGGER.info('Enable failover called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.enable_failover() + dhcp_config.write_config() + LOGGER.info('Failover enabled') + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to enable failover: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def GetDHCPRange(self, request, context): # pylint: disable=W0613 """ Resolve the current DHCP configuration and return the first range from the first subnet in the file """ - self._dhcp_config.resolve_config() - pool = self._dhcp_config.subnets[0].pools[0] - return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + LOGGER.info('Get DHCP range called') + try: + pool = self._get_dhcp_config()._subnets[0].pools[0] + return pb2.DHCPRange(code=200, start=pool.range_start, end=pool.range_end) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) - def SetDHCPRange(self, request, context): # pylint: disable=W0613 + def GetLease(self, request, context): # pylint: disable=W0613 + """ + Resolve the current DHCP leased address for the + provided MAC address + """ + LOGGER.info('Get lease called') + try: + lease = self.dhcp_leases.get_lease(request.hw_addr) + if lease is not None: + return pb2.Response(code=200, message=str(lease)) + else: + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to get lease: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def SetDHCPRange(self, request, context): # pylint: disable=W0613 """ Change DHCP configuration and set the the first range from the first subnet in the configuration """ + LOGGER.info('Set DHCP range called') + try: + dhcp_config = self._get_dhcp_config() + dhcp_config.set_range(request.start, request.end, 0, 0) + dhcp_config.write_config() + LOGGER.info('DHCP range set') + return pb2.Response(code=200, message='DHCP Range Set') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to set DHCP range: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) - print('Setting DHCPRange') - print('Start: ' + request.start) - print('End: ' + request.end) - self._dhcp_config.resolve_config() - self._dhcp_config.set_range(request.start, request.end, 0, 0) - self._dhcp_config.write_config() - return pb2.Response(code=200, message='DHCP Range Set') - - def GetStatus(self, request, context): # pylint: disable=W0613 + def GetStatus(self, request, context): # pylint: disable=W0613 """ Return the current status of the network module """ diff --git a/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto index 8e2732620..b6a11a75b 100644 --- a/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto +++ b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto @@ -2,16 +2,49 @@ syntax = "proto3"; service NetworkModule { + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; + + rpc DeleteReservedLease(DeleteReservedLeaseRequest) returns (Response) {}; + + rpc DisableFailover(DisableFailoverRequest) returns (Response) {}; + + rpc EnableFailover(EnableFailoverRequest) returns (Response) {}; + rpc GetDHCPRange(GetDHCPRangeRequest) returns (DHCPRange) {}; - rpc SetDHCPRange(DHCPRange) returns (Response) {}; + rpc GetLease(GetLeaseRequest) returns (Response) {}; rpc GetStatus(GetStatusRequest) returns (Response) {}; - rpc GetIPAddress(GetIPAddressRequest) returns (Response) {}; + rpc SetDHCPRange(SetDHCPRangeRequest) returns (Response) {}; +} + +message AddReservedLeaseRequest { + string hostname = 1; + string hw_addr = 2; + string ip_addr = 3; +} + +message DeleteReservedLeaseRequest { + string hw_addr = 1; +} + +message DisableFailoverRequest {} + +message EnableFailoverRequest {} + +message GetDHCPRangeRequest {} + +message GetLeaseRequest { + string hw_addr = 1; +} - rpc SetLeaseAddress(SetLeaseAddressRequest) returns (Response) {}; +message GetStatusRequest {} +message SetDHCPRangeRequest { + int32 code = 1; + string start = 2; + string end = 3; } message Response { @@ -20,17 +53,7 @@ message Response { } message DHCPRange { - int32 code = 1; + int32 code = 1; string start = 2; string end = 3; } - -message GetDHCPRangeRequest {} - -message GetIPAddressRequest {} - -message GetStatusRequest {} - -message SetLeaseAddressRequest { - string ipAddress = 1; -} \ No newline at end of file From 1b3f13f4e24b0a43be1482b41a16eb94cb9b5a7e Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 3 Jul 2023 13:43:52 -0600 Subject: [PATCH 20/25] fix line endings --- cmd/start | 76 +++++++++++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/cmd/start b/cmd/start index 26b1d5a0d..64ac197eb 100755 --- a/cmd/start +++ b/cmd/start @@ -1,39 +1,39 @@ -#!/bin/bash -e - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root. Use sudo cmd/start" - exit 1 -fi - -# Ensure that /var/run/netns folder exists -mkdir -p /var/run/netns - -# Clear up existing runtime files -rm -rf runtime - -# Check if python modules exist. Install if not -[ ! -d "venv" ] && cmd/install - -# Activate Python virtual environment -source venv/bin/activate - -# TODO: Execute python code -# Set the PYTHONPATH to include the "src" directory -export PYTHONPATH="$PWD/framework/python/src" -python -u framework/python/src/core/test_runner.py $@ - +#!/bin/bash -e + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "$EUID" -ne 0 ]]; then + echo "Must run as root. Use sudo cmd/start" + exit 1 +fi + +# Ensure that /var/run/netns folder exists +mkdir -p /var/run/netns + +# Clear up existing runtime files +rm -rf runtime + +# Check if python modules exist. Install if not +[ ! -d "venv" ] && cmd/install + +# Activate Python virtual environment +source venv/bin/activate + +# TODO: Execute python code +# Set the PYTHONPATH to include the "src" directory +export PYTHONPATH="$PWD/framework/python/src" +python -u framework/python/src/core/test_runner.py $@ + deactivate \ No newline at end of file From d4e3e9102f7eafc662989fbf897be1722bc202f8 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Mon, 3 Jul 2023 14:03:56 -0600 Subject: [PATCH 21/25] misc cleanup --- modules/test/base/bin/start_module | 4 +-- .../test/conn/python/src/connection_module.py | 34 ++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/modules/test/base/bin/start_module b/modules/test/base/bin/start_module index 2f73a340b..82c9d26bf 100644 --- a/modules/test/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -57,10 +57,10 @@ then exit 1 fi -echo "Setting up PYTHONPATH..." # Setup the PYTHONPATH so all imports work as expected -echo "PYTHONPATH: $PYTHONPATH" +echo "Setting up PYTHONPATH..." export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" # Build all gRPC files from the proto for use in # gRPC clients for communications to network modules diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index c2600335b..a1727df23 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -36,29 +36,31 @@ def __init__(self, module): LOGGER = self._get_logger() self.dhcp1_client = DHCPClient1() - response = self.dhcp1_client.add_reserved_lease('test','00:11:22:33:44:55','10.10.10.21') - print("AddLeaseResp: " + str(response)) + # ToDo: Move this into some level of testing, leave for + # reference until tests are implemented with these calls + # response = self.dhcp1_client.add_reserved_lease('test','00:11:22:33:44:55','10.10.10.21') + # print("AddLeaseResp: " + str(response)) - response = self.dhcp1_client.delete_reserved_lease('00:11:22:33:44:55') - print("DelLeaseResp: " + str(response)) + # response = self.dhcp1_client.delete_reserved_lease('00:11:22:33:44:55') + # print("DelLeaseResp: " + str(response)) - response = self.dhcp1_client.disable_failover() - print("FailoverDisabled: " + str(response)) + # response = self.dhcp1_client.disable_failover() + # print("FailoverDisabled: " + str(response)) - response = self.dhcp1_client.enable_failover() - print("FailoverEnabled: " + str(response)) + # response = self.dhcp1_client.enable_failover() + # print("FailoverEnabled: " + str(response)) - response = self.dhcp1_client.get_dhcp_range() - print("DHCP Range: " + str(response)) + # response = self.dhcp1_client.get_dhcp_range() + # print("DHCP Range: " + str(response)) - response = self.dhcp1_client.get_lease(self._device_mac) - print("Lease: " + str(response)) + # response = self.dhcp1_client.get_lease(self._device_mac) + # print("Lease: " + str(response)) - response = self.dhcp1_client.get_status() - print("Status: " + str(response)) + # response = self.dhcp1_client.get_status() + # print("Status: " + str(response)) - response = self.dhcp1_client.set_dhcp_range('10.10.10.20','10.10.10.30') - print("Set Range: " + str(response)) + # response = self.dhcp1_client.set_dhcp_range('10.10.10.20','10.10.10.30') + # print("Set Range: " + str(response)) def _connection_mac_address(self): From 51da71fc813fc850205e7a8e8474bb142a528eb6 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Wed, 5 Jul 2023 22:09:33 -0600 Subject: [PATCH 22/25] Move isc-dhcp-server and radvd to services Move DHCP server monitoring and booting to python script --- modules/network/dhcp-1/bin/radvd-service | 41 +++++++ .../network/dhcp-1/bin/start_network_service | 56 ++------- modules/network/dhcp-1/conf/isc-dhcp-server | 4 + modules/network/dhcp-1/dhcp-1.Dockerfile | 2 +- .../python/src/grpc_server/dhcp_server.py | 106 ++++++++++++++++++ .../python/src/grpc_server/radvd_server.py | 55 +++++++++ 6 files changed, 215 insertions(+), 49 deletions(-) create mode 100644 modules/network/dhcp-1/bin/radvd-service create mode 100644 modules/network/dhcp-1/conf/isc-dhcp-server create mode 100644 modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py create mode 100644 modules/network/dhcp-1/python/src/grpc_server/radvd_server.py diff --git a/modules/network/dhcp-1/bin/radvd-service b/modules/network/dhcp-1/bin/radvd-service new file mode 100644 index 000000000..671e0fae2 --- /dev/null +++ b/modules/network/dhcp-1/bin/radvd-service @@ -0,0 +1,41 @@ +#!/bin/bash + +RA_PID_FILE=/var/run/radvd/radvd.pid +RA_LOG_FILE=/runtime/network/dhcp1-radvd.log + +stop_radvd() { + # Directly kill by PID file reference + kill -9 $(cat $RA_PID_FILE) || true + rm -f $RA_PID_FILE + # Generic kill by process name just in case PID didn't work + pkill radvd +} + +start_radvd(){ + /usr/sbin/radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE +} + +case "$1" in + start) + start_radvd + ;; + stop) + stop_radvd + ;; + restart) + stop_radvd + sleep 1 + start_radvd + ;; + status) + if pgrep radvd > /dev/null; then + echo "radvd service is running." + else + echo "radvd service is not running." + fi + ;; + *) + echo "Usage: $0 {start|stop|restart}" + exit 1 + ;; +esac \ No newline at end of file diff --git a/modules/network/dhcp-1/bin/start_network_service b/modules/network/dhcp-1/bin/start_network_service index 9f4a3dc51..82b4c6e33 100644 --- a/modules/network/dhcp-1/bin/start_network_service +++ b/modules/network/dhcp-1/bin/start_network_service @@ -29,63 +29,23 @@ sysctl -p # Create leases file if needed touch /var/lib/dhcp/dhcpd.leases -#Create directory for radvd +# Create directory for radvd mkdir /var/run/radvd -#Create and set permissions on the log files +# Create and set permissions on the log files touch $DHCP_LOG_FILE touch $RA_LOG_FILE chown $HOST_USER $DHCP_LOG_FILE chown $HOST_USER $RA_LOG_FILE -#Move the config files to the correct location +# Move the config files to the correct location +cp /testrun/conf/isc-dhcp-server /etc/default/ cp /testrun/conf/dhcpd.conf /etc/dhcp/dhcpd.conf cp /testrun/conf/radvd.conf /etc/radvd.conf -# Restart dhcp server when config changes -while true; do +# Move the radvd-sevice file to the correct location +cp /testrun/bin/radvd-service /usr/local/bin/ - new_checksum=$(md5sum $CONFIG_FILE) - - if [ "$checksum" == "$new_checksum" ]; then - sleep 2 - continue - fi - - echo Config changed. Restarting dhcp server at $(date).. - - if [ -f $DHCP_PID_FILE ]; then - kill -9 $(cat $DHCP_PID_FILE) || true - rm -f $DHCP_PID_FILE - fi - - if [ -f $RA_PID_FILE ]; then - kill -9 $(cat $RA_PID_FILE) || true - rm -f $RA_PID_FILE - fi - - checksum=$new_checksum - - echo Starting isc-dhcp-server at $(date) - - radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE - dhcpd -d &> $DHCP_LOG_FILE & - - while [ ! -f $DHCP_PID_FILE ]; do - echo Waiting for $DHCP_PID_FILE... - sleep 2 - done - - echo $DHCP_PID_FILE now available - - while [ ! -f $RA_PID_FILE ]; do - echo Waiting for $RA_PID_FILE... - sleep 2 - done - - echo $RA_PID_FILE now available - - echo Server now stable - -done \ No newline at end of file +# Start the DHCP Server +python3 -u /testrun/python/src/grpc_server/dhcp_server.py \ No newline at end of file diff --git a/modules/network/dhcp-1/conf/isc-dhcp-server b/modules/network/dhcp-1/conf/isc-dhcp-server new file mode 100644 index 000000000..44db95cd9 --- /dev/null +++ b/modules/network/dhcp-1/conf/isc-dhcp-server @@ -0,0 +1,4 @@ +# On what interfaces should the DHCP server (dhcpd) serve DHCP requests? +# Separate multiple interfaces with spaces, e.g. "eth0 eth1". +INTERFACESv4="veth0" +#INTERFACESv6="veth0" diff --git a/modules/network/dhcp-1/dhcp-1.Dockerfile b/modules/network/dhcp-1/dhcp-1.Dockerfile index b47378045..6b941d878 100644 --- a/modules/network/dhcp-1/dhcp-1.Dockerfile +++ b/modules/network/dhcp-1/dhcp-1.Dockerfile @@ -25,7 +25,7 @@ RUN apt-get install -y wget RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ # Install dhcp server -RUN apt-get install -y isc-dhcp-server radvd +RUN apt-get install -y isc-dhcp-server radvd systemd # Copy over all configuration files COPY $MODULE_DIR/conf /testrun/conf diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py new file mode 100644 index 000000000..d3b147671 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py @@ -0,0 +1,106 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server""" +import time +from common import logger +from common import util +from dhcp_config import DHCPConfig +from radvd_server import RADVDServer + +CONFIG_FILE = '/etc/dhcp/dhcpd.conf' +LOG_NAME = 'dhcp_server' +LOGGER = None + + +class DHCPServer: + """Represents the DHCP Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + self.dhcp_config = DHCPConfig() + self.radvd = RADVDServer() + self.dhcp_config.resolve_config() + + def restart(self): + LOGGER.info("Restarting DHCP Server") + response = util.run_command("service isc-dhcp-server restart", False) + LOGGER.info("DHCP Restarted: " + str(response)) + return response + + def start(self): + LOGGER.info("Starting DHCP Server") + response = util.run_command("service isc-dhcp-server start", False) + LOGGER.info("DHCP Started: " + str(response)) + return response + + def stop(self): + LOGGER.info("Stopping DHCP Server") + response = util.run_command("service isc-dhcp-server stop", False) + LOGGER.info("DHCP Stopped: " + str(response)) + + def is_running(self): + LOGGER.info("Checking DHCP Status") + response = util.run_command("service isc-dhcp-server status") + LOGGER.info("DHCP Status: " + str(response)) + return response[0] == 'Status of ISC DHCPv4 server: dhcpd is running.' + + def boot(self): + LOGGER.info("Booting DHCP Server") + isc_booted = False + radvd_booted = False + if self.is_running(): + LOGGER.info("Stopping isc-dhcp-server") + stopped = self.stop() + LOGGER.info("isc-dhcp-server stopped: " + str(stopped)) + + if self.radvd.is_running(): + LOGGER.info("Stopping RADVD") + stopped = self.radvd.stop() + LOGGER.info("radvd stopped: " + str(stopped)) + + LOGGER.info("Starting isc-dhcp-server") + if self.start(): + isc_booted = self.is_running() + LOGGER.info("isc-dhcp-server started: " + str(isc_booted)) + + LOGGER.info("Starting RADVD") + if self.radvd.start(): + radvd_booted = self.radvd.is_running() + LOGGER.info("RADVD started: " + str(radvd_booted)) + + return isc_booted and radvd_booted + +def run(): + dhcp_server = DHCPServer() + booted = dhcp_server.boot() + + if not booted: + LOGGER.error('DHCP Server Failed to boot. Exiting') + sys.exit(1) + + config = str(dhcp_server.dhcp_config) + while True: + dhcp_server.dhcp_config.resolve_config() + new_config = str(dhcp_server.dhcp_config) + if config != new_config: + LOGGER.info("DHCP Config Changed") + config = new_config + success = dhcp_server.restart() + success = dhcp_server.radvd.restart() + time.sleep(1) + +if __name__ == '__main__': + run() diff --git a/modules/network/dhcp-1/python/src/grpc_server/radvd_server.py b/modules/network/dhcp-1/python/src/grpc_server/radvd_server.py new file mode 100644 index 000000000..48e063e61 --- /dev/null +++ b/modules/network/dhcp-1/python/src/grpc_server/radvd_server.py @@ -0,0 +1,55 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server""" +import time +from common import logger +from common import util +from dhcp_config import DHCPConfig + +CONFIG_FILE = '/etc/dhcp/dhcpd.conf' +LOG_NAME = 'radvd' +LOGGER = None + + +class RADVDServer: + """Represents the RADVD Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + + def restart(self): + LOGGER.info("Restarting RADVD Server") + response = util.run_command("radvd-service restart", False) + LOGGER.info("RADVD Restarted: " + str(response)) + return response + + def start(self): + LOGGER.info("Starting RADVD Server") + response = util.run_command("radvd-service start", False) + LOGGER.info("RADVD Started: " + str(response)) + return response + + def stop(self): + LOGGER.info("Stopping RADVD Server") + response = util.run_command("radvd-service stop", False) + LOGGER.info("RADVD Stopped: " + str(response)) + return response + + def is_running(self): + LOGGER.info("Checking RADVD Status") + response = util.run_command("radvd-service status") + LOGGER.info("RADVD Status: " + str(response)) + return response[0] == 'radvd service is running.' From 48b9f7103c955c135896195b872626a278244441 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 6 Jul 2023 00:29:24 -0600 Subject: [PATCH 23/25] Add grpc methods to interact with dhcp_server module Update dhcp_server to control radvd server directly from calls Fix radvd service status method --- modules/network/dhcp-1/bin/radvd-service | 2 +- .../python/src/grpc_server/dhcp_server.py | 48 ++++++++++++++----- .../python/src/grpc_server/network_service.py | 41 +++++++++++++++- .../python/src/grpc_server/proto/grpc.proto | 12 +++++ 4 files changed, 88 insertions(+), 15 deletions(-) diff --git a/modules/network/dhcp-1/bin/radvd-service b/modules/network/dhcp-1/bin/radvd-service index 671e0fae2..3a1559781 100644 --- a/modules/network/dhcp-1/bin/radvd-service +++ b/modules/network/dhcp-1/bin/radvd-service @@ -28,7 +28,7 @@ case "$1" in start_radvd ;; status) - if pgrep radvd > /dev/null; then + if [ -f "$RA_PID_FILE" ]; then echo "radvd service is running." else echo "radvd service is not running." diff --git a/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py b/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py index d3b147671..2f67b0c2d 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py +++ b/modules/network/dhcp-1/python/src/grpc_server/dhcp_server.py @@ -36,26 +36,36 @@ def __init__(self): def restart(self): LOGGER.info("Restarting DHCP Server") - response = util.run_command("service isc-dhcp-server restart", False) - LOGGER.info("DHCP Restarted: " + str(response)) - return response + isc_started = util.run_command("service isc-dhcp-server restart", False) + radvd_started = self.radvd.restart() + started = isc_started and radvd_started + LOGGER.info("DHCP Restarted: " + str(started)) + return started def start(self): LOGGER.info("Starting DHCP Server") - response = util.run_command("service isc-dhcp-server start", False) - LOGGER.info("DHCP Started: " + str(response)) - return response + isc_started = util.run_command("service isc-dhcp-server start", False) + radvd_started = self.radvd.start() + started = isc_started and radvd_started + LOGGER.info("DHCP Started: " + str(started)) + return started def stop(self): LOGGER.info("Stopping DHCP Server") - response = util.run_command("service isc-dhcp-server stop", False) - LOGGER.info("DHCP Stopped: " + str(response)) + isc_stopped = util.run_command("service isc-dhcp-server stop", False) + radvd_stopped = self.radvd.stop() + stopped = isc_stopped and radvd_stopped + LOGGER.info("DHCP Stopped: " + str(stopped)) + return stopped def is_running(self): LOGGER.info("Checking DHCP Status") response = util.run_command("service isc-dhcp-server status") - LOGGER.info("DHCP Status: " + str(response)) - return response[0] == 'Status of ISC DHCPv4 server: dhcpd is running.' + isc_running = response[0] == 'Status of ISC DHCPv4 server: dhcpd is running.' + radvd_running = self.radvd.is_running() + running = isc_running and radvd_running + LOGGER.info("DHCP Status: " + str(running)) + return running def boot(self): LOGGER.info("Booting DHCP Server") @@ -73,14 +83,28 @@ def boot(self): LOGGER.info("Starting isc-dhcp-server") if self.start(): - isc_booted = self.is_running() + isc_booted = False + # Scan for 5 seconds if not yet ready + for i in range(5): + time.sleep(1) + isc_booted = self.is_running() + if isc_booted: + break; LOGGER.info("isc-dhcp-server started: " + str(isc_booted)) LOGGER.info("Starting RADVD") if self.radvd.start(): - radvd_booted = self.radvd.is_running() + radvd_booted = False + # Scan for 5 seconds if not yet ready + for i in range(5): + time.sleep(1) + radvd_booted = self.radvd.is_running() + if radvd_booted: + break; LOGGER.info("RADVD started: " + str(radvd_booted)) + + return isc_booted and radvd_booted def run(): diff --git a/modules/network/dhcp-1/python/src/grpc_server/network_service.py b/modules/network/dhcp-1/python/src/grpc_server/network_service.py index bf2b98803..813e2dd21 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/network_service.py +++ b/modules/network/dhcp-1/python/src/grpc_server/network_service.py @@ -15,6 +15,7 @@ import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 +from dhcp_server import DHCPServer from dhcp_config import DHCPConfig from dhcp_leases import DHCPLeases @@ -28,6 +29,7 @@ class NetworkService(pb2_grpc.NetworkModule): """gRPC endpoints for the DHCP Server""" def __init__(self): + self._dhcp_server = DHCPServer() self._dhcp_config = None self.dhcp_leases = DHCPLeases() global LOGGER @@ -39,6 +41,42 @@ def _get_dhcp_config(self): self._dhcp_config.resolve_config() return self._dhcp_config + def RestartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Restarting DHCP server') + try: + started = self._dhcp_server.restart() + LOGGER.info('DHCP server restarted: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to restart DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Starting DHCP server') + try: + started = self._dhcp_server.start() + LOGGER.info('DHCP server started: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to start DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StopDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Stopping DHCP server') + try: + stopped = self._dhcp_server.stop() + LOGGER.info('DHCP server stopped: ' + (str(stopped))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to stop DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + def AddReservedLease(self, request, context): # pylint: disable=W0613 LOGGER.info('Add reserved lease called') try: @@ -151,7 +189,6 @@ def GetStatus(self, request, context): # pylint: disable=W0613 """ Return the current status of the network module """ - # ToDo: Figure out how to resolve the current DHCP status - dhcp_status = True + dhcp_status = self._dhcp_server.is_running() message = str({'dhcpStatus': dhcp_status}) return pb2.Response(code=200, message=message) diff --git a/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto index d9f56213e..ad454ad30 100644 --- a/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto +++ b/modules/network/dhcp-1/python/src/grpc_server/proto/grpc.proto @@ -2,6 +2,12 @@ syntax = "proto3"; service NetworkModule { + rpc RestartDHCPServer(RestartDHCPServerRequest) returns (Response) {}; + + rpc StartDHCPServer(StartDHCPServerRequest) returns (Response) {}; + + rpc StopDHCPServer(StopDHCPServerRequest) returns (Response) {}; + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; rpc DeleteReservedLease(DeleteReservedLeaseRequest) returns (Response) {}; @@ -29,6 +35,12 @@ message DeleteReservedLeaseRequest { string hw_addr = 1; } +message RestartDHCPServerRequest {} + +message StartDHCPServerRequest {} + +message StopDHCPServerRequest {} + message DisableFailoverRequest {} message EnableFailoverRequest {} From 020ca9e44a20f5171460d4333cc6bb7229ed2aab Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 6 Jul 2023 01:37:09 -0600 Subject: [PATCH 24/25] Add updates to dhcp2 module Update radvd service --- modules/network/dhcp-1/bin/radvd-service | 12 +- modules/network/dhcp-2/bin/radvd-service | 41 ++++++ .../network/dhcp-2/bin/start_network_service | 56 ++------ modules/network/dhcp-2/conf/isc-dhcp-server | 4 + modules/network/dhcp-2/dhcp-2.Dockerfile | 11 +- .../python/src/grpc_server/dhcp_config.py | 2 +- .../python/src/grpc_server/dhcp_server.py | 130 ++++++++++++++++++ .../python/src/grpc_server/network_service.py | 41 +++++- .../python/src/grpc_server/proto/grpc.proto | 12 ++ .../python/src/grpc_server/radvd_server.py | 55 ++++++++ 10 files changed, 304 insertions(+), 60 deletions(-) create mode 100644 modules/network/dhcp-2/bin/radvd-service create mode 100644 modules/network/dhcp-2/conf/isc-dhcp-server create mode 100644 modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py create mode 100644 modules/network/dhcp-2/python/src/grpc_server/radvd_server.py diff --git a/modules/network/dhcp-1/bin/radvd-service b/modules/network/dhcp-1/bin/radvd-service index 3a1559781..cd55956e7 100644 --- a/modules/network/dhcp-1/bin/radvd-service +++ b/modules/network/dhcp-1/bin/radvd-service @@ -3,12 +3,12 @@ RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp1-radvd.log -stop_radvd() { +stop_radvd(){ # Directly kill by PID file reference - kill -9 $(cat $RA_PID_FILE) || true - rm -f $RA_PID_FILE - # Generic kill by process name just in case PID didn't work - pkill radvd + if [ -f "$RA_PID_FILE" ]; then + kill -9 $(cat $RA_PID_FILE) || true + rm -f $RA_PID_FILE + fi } start_radvd(){ @@ -35,7 +35,7 @@ case "$1" in fi ;; *) - echo "Usage: $0 {start|stop|restart}" + echo "Usage: $0 {start|stop|status|restart}" exit 1 ;; esac \ No newline at end of file diff --git a/modules/network/dhcp-2/bin/radvd-service b/modules/network/dhcp-2/bin/radvd-service new file mode 100644 index 000000000..41bf5d772 --- /dev/null +++ b/modules/network/dhcp-2/bin/radvd-service @@ -0,0 +1,41 @@ +#!/bin/bash + +RA_PID_FILE=/var/run/radvd/radvd.pid +RA_LOG_FILE=/runtime/network/dhcp2-radvd.log + +stop_radvd(){ + # Directly kill by PID file reference + if [ -f "$RA_PID_FILE" ]; then + kill -9 $(cat $RA_PID_FILE) || true + rm -f $RA_PID_FILE + fi +} + +start_radvd(){ + /usr/sbin/radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE +} + +case "$1" in + start) + start_radvd + ;; + stop) + stop_radvd + ;; + restart) + stop_radvd + sleep 1 + start_radvd + ;; + status) + if [ -f "$RA_PID_FILE" ]; then + echo "radvd service is running." + else + echo "radvd service is not running." + fi + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac \ No newline at end of file diff --git a/modules/network/dhcp-2/bin/start_network_service b/modules/network/dhcp-2/bin/start_network_service index 723689278..ed7d3125e 100644 --- a/modules/network/dhcp-2/bin/start_network_service +++ b/modules/network/dhcp-2/bin/start_network_service @@ -29,63 +29,23 @@ sysctl -p # Create leases file if needed touch /var/lib/dhcp/dhcpd.leases -#Create directory for radvd +# Create directory for radvd mkdir /var/run/radvd -#Create and set permissions on the log files +# Create and set permissions on the log files touch $DHCP_LOG_FILE touch $RA_LOG_FILE chown $HOST_USER $DHCP_LOG_FILE chown $HOST_USER $RA_LOG_FILE -#Move the config files to the correct location +# Move the config files to the correct location +cp /testrun/conf/isc-dhcp-server /etc/default/ cp /testrun/conf/dhcpd.conf /etc/dhcp/dhcpd.conf cp /testrun/conf/radvd.conf /etc/radvd.conf -# Restart dhcp server when config changes -while true; do +# Move the radvd-sevice file to the correct location +cp /testrun/bin/radvd-service /usr/local/bin/ - new_checksum=$(md5sum $CONFIG_FILE) - - if [ "$checksum" == "$new_checksum" ]; then - sleep 2 - continue - fi - - echo Config changed. Restarting dhcp server at $(date).. - - if [ -f $DHCP_PID_FILE ]; then - kill -9 $(cat $DHCP_PID_FILE) || true - rm -f $DHCP_PID_FILE - fi - - if [ -f $RA_PID_FILE ]; then - kill -9 $(cat $RA_PID_FILE) || true - rm -f $RA_PID_FILE - fi - - checksum=$new_checksum - - echo Starting isc-dhcp-server at $(date) - - radvd -m logfile -l $RA_LOG_FILE -p $RA_PID_FILE - dhcpd -d &> $DHCP_LOG_FILE & - - while [ ! -f $DHCP_PID_FILE ]; do - echo Waiting for $DHCP_PID_FILE... - sleep 2 - done - - echo $DHCP_PID_FILE now available - - while [ ! -f $RA_PID_FILE ]; do - echo Waiting for $RA_PID_FILE... - sleep 2 - done - - echo $RA_PID_FILE now available - - echo Server now stable - -done \ No newline at end of file +# Start the DHCP Server +python3 -u /testrun/python/src/grpc_server/dhcp_server.py \ No newline at end of file diff --git a/modules/network/dhcp-2/conf/isc-dhcp-server b/modules/network/dhcp-2/conf/isc-dhcp-server new file mode 100644 index 000000000..44db95cd9 --- /dev/null +++ b/modules/network/dhcp-2/conf/isc-dhcp-server @@ -0,0 +1,4 @@ +# On what interfaces should the DHCP server (dhcpd) serve DHCP requests? +# Separate multiple interfaces with spaces, e.g. "eth0 eth1". +INTERFACESv4="veth0" +#INTERFACESv6="veth0" diff --git a/modules/network/dhcp-2/dhcp-2.Dockerfile b/modules/network/dhcp-2/dhcp-2.Dockerfile index df77cb811..153aa50e7 100644 --- a/modules/network/dhcp-2/dhcp-2.Dockerfile +++ b/modules/network/dhcp-2/dhcp-2.Dockerfile @@ -18,8 +18,14 @@ FROM test-run/base:latest ARG MODULE_NAME=dhcp-2 ARG MODULE_DIR=modules/network/$MODULE_NAME +# Install all necessary packages +RUN apt-get install -y wget + +#Update the oui.txt file from ieee +RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ + # Install dhcp server -RUN apt-get install -y isc-dhcp-server radvd +RUN apt-get install -y isc-dhcp-server radvd systemd # Copy over all configuration files COPY $MODULE_DIR/conf /testrun/conf @@ -28,5 +34,4 @@ COPY $MODULE_DIR/conf /testrun/conf COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY $MODULE_DIR/python /testrun/python - +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py index 444faa87c..f796b7fad 100644 --- a/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_config.py @@ -33,7 +33,7 @@ def __init__(self): self._peer = None self._reserved_hosts = [] global LOGGER - LOGGER = logger.get_logger(LOG_NAME, 'dhcp-1') + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') def add_reserved_host(self, hostname, hw_addr, ip_addr): host = DHCPReservedHost(hostname=hostname, diff --git a/modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py b/modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py new file mode 100644 index 000000000..1431d6ddd --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/dhcp_server.py @@ -0,0 +1,130 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server""" +import time +from common import logger +from common import util +from dhcp_config import DHCPConfig +from radvd_server import RADVDServer + +CONFIG_FILE = '/etc/dhcp/dhcpd.conf' +LOG_NAME = 'dhcp_server' +LOGGER = None + + +class DHCPServer: + """Represents the DHCP Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + self.dhcp_config = DHCPConfig() + self.radvd = RADVDServer() + self.dhcp_config.resolve_config() + + def restart(self): + LOGGER.info("Restarting DHCP Server") + isc_started = util.run_command("service isc-dhcp-server restart", False) + radvd_started = self.radvd.restart() + started = isc_started and radvd_started + LOGGER.info("DHCP Restarted: " + str(started)) + return started + + def start(self): + LOGGER.info("Starting DHCP Server") + isc_started = util.run_command("service isc-dhcp-server start", False) + radvd_started = self.radvd.start() + started = isc_started and radvd_started + LOGGER.info("DHCP Started: " + str(started)) + return started + + def stop(self): + LOGGER.info("Stopping DHCP Server") + isc_stopped = util.run_command("service isc-dhcp-server stop", False) + radvd_stopped = self.radvd.stop() + stopped = isc_stopped and radvd_stopped + LOGGER.info("DHCP Stopped: " + str(stopped)) + return stopped + + def is_running(self): + LOGGER.info("Checking DHCP Status") + response = util.run_command("service isc-dhcp-server status") + isc_running = response[0] == 'Status of ISC DHCPv4 server: dhcpd is running.' + radvd_running = self.radvd.is_running() + running = isc_running and radvd_running + LOGGER.info("DHCP Status: " + str(running)) + return running + + def boot(self): + LOGGER.info("Booting DHCP Server") + isc_booted = False + radvd_booted = False + if self.is_running(): + LOGGER.info("Stopping isc-dhcp-server") + stopped = self.stop() + LOGGER.info("isc-dhcp-server stopped: " + str(stopped)) + + if self.radvd.is_running(): + LOGGER.info("Stopping RADVD") + stopped = self.radvd.stop() + LOGGER.info("radvd stopped: " + str(stopped)) + + LOGGER.info("Starting isc-dhcp-server") + if self.start(): + isc_booted = False + # Scan for 5 seconds if not yet ready + for i in range(5): + time.sleep(1) + isc_booted = self.is_running() + if isc_booted: + break; + LOGGER.info("isc-dhcp-server started: " + str(isc_booted)) + + LOGGER.info("Starting RADVD") + if self.radvd.start(): + radvd_booted = False + # Scan for 5 seconds if not yet ready + for i in range(5): + time.sleep(1) + radvd_booted = self.radvd.is_running() + if radvd_booted: + break; + LOGGER.info("RADVD started: " + str(radvd_booted)) + + + + return isc_booted and radvd_booted + +def run(): + dhcp_server = DHCPServer() + booted = dhcp_server.boot() + + if not booted: + LOGGER.error('DHCP Server Failed to boot. Exiting') + sys.exit(1) + + config = str(dhcp_server.dhcp_config) + while True: + dhcp_server.dhcp_config.resolve_config() + new_config = str(dhcp_server.dhcp_config) + if config != new_config: + LOGGER.info("DHCP Config Changed") + config = new_config + success = dhcp_server.restart() + success = dhcp_server.radvd.restart() + time.sleep(1) + +if __name__ == '__main__': + run() diff --git a/modules/network/dhcp-2/python/src/grpc_server/network_service.py b/modules/network/dhcp-2/python/src/grpc_server/network_service.py index 053d26d6b..19dfb775c 100644 --- a/modules/network/dhcp-2/python/src/grpc_server/network_service.py +++ b/modules/network/dhcp-2/python/src/grpc_server/network_service.py @@ -15,6 +15,7 @@ import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 +from dhcp_server import DHCPServer from dhcp_config import DHCPConfig from dhcp_leases import DHCPLeases @@ -28,6 +29,7 @@ class NetworkService(pb2_grpc.NetworkModule): """gRPC endpoints for the DHCP Server""" def __init__(self): + self._dhcp_server = DHCPServer() self._dhcp_config = None self.dhcp_leases = DHCPLeases() global LOGGER @@ -39,6 +41,42 @@ def _get_dhcp_config(self): self._dhcp_config.resolve_config() return self._dhcp_config + def RestartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Restarting DHCP server') + try: + started = self._dhcp_server.restart() + LOGGER.info('DHCP server restarted: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to restart DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StartDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Starting DHCP server') + try: + started = self._dhcp_server.start() + LOGGER.info('DHCP server started: ' + (str(started))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to start DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + + def StopDHCPServer(self, request, context): # pylint: disable=W0613 + LOGGER.info('Stopping DHCP server') + try: + stopped = self._dhcp_server.stop() + LOGGER.info('DHCP server stopped: ' + (str(stopped))) + return pb2.Response(code=200, message='{}') + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to stop DHCP server: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.Response(code=500, message=fail_message) + def AddReservedLease(self, request, context): # pylint: disable=W0613 LOGGER.info('Add reserved lease called') try: @@ -151,7 +189,6 @@ def GetStatus(self, request, context): # pylint: disable=W0613 """ Return the current status of the network module """ - # ToDo: Figure out how to resolve the current DHCP status - dhcp_status = True + dhcp_status = self._dhcp_server.is_running() message = str({'dhcpStatus': dhcp_status}) return pb2.Response(code=200, message=message) diff --git a/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto index b6a11a75b..f2e19fd5f 100644 --- a/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto +++ b/modules/network/dhcp-2/python/src/grpc_server/proto/grpc.proto @@ -2,6 +2,12 @@ syntax = "proto3"; service NetworkModule { + rpc RestartDHCPServer(RestartDHCPServerRequest) returns (Response) {}; + + rpc StartDHCPServer(StartDHCPServerRequest) returns (Response) {}; + + rpc StopDHCPServer(StopDHCPServerRequest) returns (Response) {}; + rpc AddReservedLease(AddReservedLeaseRequest) returns (Response) {}; rpc DeleteReservedLease(DeleteReservedLeaseRequest) returns (Response) {}; @@ -29,6 +35,12 @@ message DeleteReservedLeaseRequest { string hw_addr = 1; } +message RestartDHCPServerRequest {} + +message StartDHCPServerRequest {} + +message StopDHCPServerRequest {} + message DisableFailoverRequest {} message EnableFailoverRequest {} diff --git a/modules/network/dhcp-2/python/src/grpc_server/radvd_server.py b/modules/network/dhcp-2/python/src/grpc_server/radvd_server.py new file mode 100644 index 000000000..0c6ef90d6 --- /dev/null +++ b/modules/network/dhcp-2/python/src/grpc_server/radvd_server.py @@ -0,0 +1,55 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains all the necessary classes to maintain the +DHCP server""" +import time +from common import logger +from common import util +from dhcp_config import DHCPConfig + +CONFIG_FILE = '/etc/dhcp/dhcpd.conf' +LOG_NAME = 'radvd' +LOGGER = None + + +class RADVDServer: + """Represents the RADVD Server""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'dhcp-2') + + def restart(self): + LOGGER.info("Restarting RADVD Server") + response = util.run_command("radvd-service restart", False) + LOGGER.info("RADVD Restarted: " + str(response)) + return response + + def start(self): + LOGGER.info("Starting RADVD Server") + response = util.run_command("radvd-service start", False) + LOGGER.info("RADVD Started: " + str(response)) + return response + + def stop(self): + LOGGER.info("Stopping RADVD Server") + response = util.run_command("radvd-service stop", False) + LOGGER.info("RADVD Stopped: " + str(response)) + return response + + def is_running(self): + LOGGER.info("Checking RADVD Status") + response = util.run_command("radvd-service status") + LOGGER.info("RADVD Status: " + str(response)) + return response[0] == 'radvd service is running.' From 896daa25ccd2635646f0431b582a6a3982a72982 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 6 Jul 2023 02:15:36 -0600 Subject: [PATCH 25/25] Add license headers --- modules/network/dhcp-1/bin/radvd-service | 14 ++++++++++++++ modules/network/dhcp-2/bin/radvd-service | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/modules/network/dhcp-1/bin/radvd-service b/modules/network/dhcp-1/bin/radvd-service index cd55956e7..1cfe499cb 100644 --- a/modules/network/dhcp-1/bin/radvd-service +++ b/modules/network/dhcp-1/bin/radvd-service @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp1-radvd.log diff --git a/modules/network/dhcp-2/bin/radvd-service b/modules/network/dhcp-2/bin/radvd-service index 41bf5d772..912c64ee3 100644 --- a/modules/network/dhcp-2/bin/radvd-service +++ b/modules/network/dhcp-2/bin/radvd-service @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + RA_PID_FILE=/var/run/radvd/radvd.pid RA_LOG_FILE=/runtime/network/dhcp2-radvd.log