diff --git a/cmd/build b/cmd/build index 6096b1909..8ecccb5ef 100755 --- a/cmd/build +++ b/cmd/build @@ -60,7 +60,6 @@ fi # Build network modules echo Building network modules -mkdir -p build/network for dir in modules/network/* ; do module=$(basename $dir) echo Building network module $module... @@ -74,7 +73,6 @@ done # Build validators echo Building network validators -mkdir -p build/devices for dir in modules/devices/* ; do module=$(basename $dir) echo Building validator module $module... @@ -88,7 +86,6 @@ done # Build test modules echo Building test modules -mkdir -p build/test for dir in modules/test/* ; do module=$(basename $dir) echo Building test module $module... diff --git a/framework/python/src/common/util.py b/framework/python/src/common/util.py index 7c31631fb..ba1b23e81 100644 --- a/framework/python/src/common/util.py +++ b/framework/python/src/common/util.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Provides basic utilities for the network orchestrator.""" +"""Provides basic utilities for Testrun.""" import getpass import os import subprocess @@ -52,12 +52,15 @@ def run_command(cmd, output=True, timeout=None): def interface_exists(interface): + """Checks whether an interface is available""" return interface in netifaces.interfaces() def prettify(mac_string): + """Formats a MAC address with colons""" return ':'.join([f'{ord(b):02x}' for b in mac_string]) def get_host_user(): + """Returns the username of the host user""" user = get_os_user() # If primary method failed, try secondary @@ -67,6 +70,7 @@ def get_host_user(): return user def get_os_user(): + """Attempts to get the username using os library""" user = None try: user = os.getlogin() @@ -79,6 +83,7 @@ def get_os_user(): return user def get_user(): + """Attempts to get the host user using the getpass library""" user = None try: user = getpass.getuser() @@ -97,9 +102,11 @@ def get_user(): return user def set_file_owner(path, owner): + """Change the owner of a file""" run_command(f'chown -R {owner} {path}') def get_module_display_name(search): + """Returns the display name of a test module""" modules = { 'ntp': 'NTP', 'dns': 'DNS', diff --git a/modules/devices/faux-dev/bin/start_network_service b/modules/devices/faux-dev/bin/start_network_service index d4bb8a92d..7d689f9dd 100644 --- a/modules/devices/faux-dev/bin/start_network_service +++ b/modules/devices/faux-dev/bin/start_network_service @@ -35,7 +35,7 @@ else INTF=$DEFINED_IFACE fi -#Create and set permissions on the output files +# Create and set permissions on the output files OUTPUT_DIR=/runtime/validation/ LOG_FILE=$OUTPUT_DIR/$MODULE_NAME.log RESULT_FILE=$OUTPUT_DIR/result.json diff --git a/modules/devices/faux-dev/faux-dev.Dockerfile b/modules/devices/faux-dev/faux-dev.Dockerfile index 9dfe1bd9b..18901a2a1 100644 --- a/modules/devices/faux-dev/faux-dev.Dockerfile +++ b/modules/devices/faux-dev/faux-dev.Dockerfile @@ -17,8 +17,9 @@ FROM testrun/base:latest ARG MODULE_NAME=faux-dev ARG MODULE_DIR=modules/devices/$MODULE_NAME +ARG COMMON_DIR=framework/python/src/common -#Update and get all additional requirements not contained in the base image +# Update and get all additional requirements not contained in the base image RUN apt-get update --fix-missing # NTP requireds interactive installation so we're going to turn that off @@ -34,4 +35,4 @@ COPY $MODULE_DIR/conf /testrun/conf COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY $MODULE_DIR/python /testrun/python \ No newline at end of file +COPY $MODULE_DIR/python /testrun/python diff --git a/modules/devices/faux-dev/python/src/logger.py b/modules/devices/faux-dev/python/src/logger.py deleted file mode 100644 index a727ad7bb..000000000 --- a/modules/devices/faux-dev/python/src/logger.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sets up the logger to be used for the faux-device.""" - -import json -import logging -import os - -LOGGERS = {} -_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' -_DATE_FORMAT = '%b %02d %H:%M:%S' -_CONF_DIR = 'conf' -_CONF_FILE_NAME = 'system.json' -_LOG_DIR = '/runtime/validation' - -# Set log level -with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), - encoding='utf-8') as conf_file: - system_conf_json = json.load(conf_file) - -log_level_str = system_conf_json['log_level'] -log_level = logging.getLevelName(log_level_str) - -log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) - - -def add_file_handler(log, log_file): - """Add file handler to existing log.""" - handler = logging.FileHandler(os.path.join(_LOG_DIR, log_file + '.log')) - handler.setFormatter(log_format) - log.addHandler(handler) - - -def add_stream_handler(log): - """Add stream handler to existing log.""" - handler = logging.StreamHandler() - handler.setFormatter(log_format) - log.addHandler(handler) - - -def get_logger(name, log_file=None): - """Return logger for requesting class.""" - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(log_level) - add_stream_handler(LOGGERS[name]) - if log_file is not None: - add_file_handler(LOGGERS[name], log_file) - return LOGGERS[name] diff --git a/modules/devices/faux-dev/python/src/util.py b/modules/devices/faux-dev/python/src/util.py deleted file mode 100644 index 81f9d2ced..000000000 --- a/modules/devices/faux-dev/python/src/util.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides basic utilities for the faux-device.""" -import subprocess -import shlex - - -def run_command(cmd, logger, output=True): - """Runs a process at the os level - By default, returns the standard output and error output - If the caller sets optional output parameter to False, - will only return a boolean result indicating if it was - successful in running the command. Failure is indicated - by any return code from the process other than zero.""" - - success = False - with subprocess.Popen( - shlex.split(cmd), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as process: - - stdout, stderr = process.communicate() - - if process.returncode != 0: - err_msg = f'{stderr.strip()}. Code: {process.returncode}' - logger.error('Command Failed: ' + cmd) - logger.error('Error: ' + err_msg) - else: - success = True - logger.debug('Command succeeded: ' + cmd) - if output: - out = stdout.strip().decode('utf-8') - logger.debug('Command output: ' + out) - return success, out - else: - return success, None diff --git a/modules/network/base/base.Dockerfile b/modules/network/base/base.Dockerfile index 0388e0793..0613e3855 100644 --- a/modules/network/base/base.Dockerfile +++ b/modules/network/base/base.Dockerfile @@ -42,5 +42,5 @@ RUN dos2unix /testrun/bin/* # Make sure all the bin files are executable RUN chmod u+x /testrun/bin/* -#Start the network module +# Start the network module ENTRYPOINT [ "/testrun/bin/start_module" ] \ No newline at end of file diff --git a/modules/network/base/bin/start_module b/modules/network/base/bin/start_module index 7b1ea8e35..3b1c092cd 100644 --- a/modules/network/base/bin/start_module +++ b/modules/network/base/bin/start_module @@ -1,105 +1,105 @@ -#!/bin/bash - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Directory where all binaries will be loaded -BIN_DIR="/testrun/bin" - -# Default interface should be veth0 for all containers -DEFAULT_IFACE=veth0 - -# Create a local user that matches the same as the host -# to be used for correct file ownership for various logs -# HOST_USER mapped in via docker container environemnt variables -useradd $HOST_USER - -# Enable IPv6 for all containers -sysctl net.ipv6.conf.all.disable_ipv6=0 -sysctl -p - -# Read in the config file -CONF_FILE="/testrun/conf/module_config.json" -CONF=`cat $CONF_FILE` - -if [[ -z $CONF ]] -then - echo "No config file present at $CONF_FILE. Exiting startup." - exit 1 -fi - -# Extract the necessary config parameters -MODULE_NAME=$(echo "$CONF" | jq -r '.config.meta.name') -DEFINED_IFACE=$(echo "$CONF" | jq -r '.config.network.interface') -HOST=$(echo "$CONF" | jq -r '.config.network.host') -GRPC=$(echo "$CONF" | jq -r '.config.grpc') - -# Validate the module name is present -if [[ -z "$MODULE_NAME" || "$MODULE_NAME" == "null" ]] -then - echo "No module name present in $CONF_FILE. Exiting startup." - exit 1 -fi - -# Select which interace to use -if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] -then - echo "No Interface Defined, defaulting to veth0" - INTF=$DEFAULT_IFACE -else - INTF=$DEFINED_IFACE -fi - -# Setup the PYTHONPATH so all imports work as expected -echo "Setting up PYTHONPATH..." -export PYTHONPATH=$($BIN_DIR/setup_python_path) -echo "PYTHONPATH: $PYTHONPATH" - -echo "Configuring binary files..." -$BIN_DIR/setup_binaries $BIN_DIR - -echo "Starting module $MODULE_NAME on local interface $INTF..." - -# Only non-host containers will have a specific -# interface for capturing -if [[ "$HOST" != "true" ]]; then - - # Wait for interface to become ready - $BIN_DIR/wait_for_interface $INTF - - # Small pause to let the interface stabalize before starting the capture - #sleep 1 - - # Start network capture - $BIN_DIR/capture $MODULE_NAME $INTF -fi - -# Start the grpc server -if [[ ! -z $GRPC && ! $GRPC == "null" ]] -then - GRPC_PORT=$(echo "$GRPC" | jq -r '.port') - if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] - then - echo "gRPC port resolved from config: $GRPC_PORT" - $BIN_DIR/start_grpc "-p $GRPC_PORT" - else - $BIN_DIR/start_grpc - fi -fi - -# Small pause to let all core services stabalize -sleep 3 - -# Start the networking service -$BIN_DIR/start_network_service $MODULE_NAME $INTF \ No newline at end of file +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Directory where all binaries will be loaded +BIN_DIR="/testrun/bin" + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Create a local user that matches the same as the host +# to be used for correct file ownership for various logs +# HOST_USER mapped in via docker container environemnt variables +useradd $HOST_USER + +# Enable IPv6 for all containers +sysctl net.ipv6.conf.all.disable_ipv6=0 +sysctl -p + +# Read in the config file +CONF_FILE="/testrun/conf/module_config.json" +CONF=`cat $CONF_FILE` + +if [[ -z $CONF ]] +then + echo "No config file present at $CONF_FILE. Exiting startup." + exit 1 +fi + +# Extract the necessary config parameters +MODULE_NAME=$(echo "$CONF" | jq -r '.config.meta.name') +DEFINED_IFACE=$(echo "$CONF" | jq -r '.config.network.interface') +HOST=$(echo "$CONF" | jq -r '.config.network.host') +GRPC=$(echo "$CONF" | jq -r '.config.grpc') + +# Validate the module name is present +if [[ -z "$MODULE_NAME" || "$MODULE_NAME" == "null" ]] +then + echo "No module name present in $CONF_FILE. Exiting startup." + exit 1 +fi + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No Interface Defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +# Setup the PYTHONPATH so all imports work as expected +echo "Setting up PYTHONPATH..." +export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" + +echo "Configuring binary files..." +$BIN_DIR/setup_binaries $BIN_DIR + +echo "Starting module $MODULE_NAME on local interface $INTF..." + +# Only non-host containers will have a specific +# interface for capturing +if [[ "$HOST" != "true" ]]; then + + # Wait for interface to become ready + $BIN_DIR/wait_for_interface $INTF + + # Small pause to let the interface stabalize before starting the capture + #sleep 1 + + # Start network capture + $BIN_DIR/capture $MODULE_NAME $INTF +fi + +# Start the grpc server +if [[ ! -z $GRPC && ! $GRPC == "null" ]] +then + GRPC_PORT=$(echo "$GRPC" | jq -r '.port') + if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] + then + echo "gRPC port resolved from config: $GRPC_PORT" + $BIN_DIR/start_grpc "-p $GRPC_PORT" + else + $BIN_DIR/start_grpc + fi +fi + +# Small pause to let all core services stabalize +sleep 3 + +# Start the networking service +$BIN_DIR/start_network_service $MODULE_NAME $INTF diff --git a/modules/network/ntp/python/src/ntp_server.py b/modules/network/ntp/python/src/ntp_server.py index 42fe21e77..fbe3ac17e 100644 --- a/modules/network/ntp/python/src/ntp_server.py +++ b/modules/network/ntp/python/src/ntp_server.py @@ -38,7 +38,7 @@ def is_running(self): if __name__ == '__main__': ntp = NTPServer() ntp.start() - # give some time for the server to start + # Give some time for the server to start running = False for _ in range(10): running = ntp.is_running() diff --git a/modules/test/base/python/src/logger.py b/modules/test/base/python/src/logger.py deleted file mode 100644 index e6a2b004c..000000000 --- a/modules/test/base/python/src/logger.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sets up the logger to be used for the test modules.""" -import json -import logging - -LOGGERS = {} -_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' -_DATE_FORMAT = '%b %02d %H:%M:%S' -_DEFAULT_LEVEL = logging.INFO -_CONF_FILE_NAME = 'testrun/system.json' -_LOG_DIR = '/runtime/output/' - -# Set log level -try: - with open(_CONF_FILE_NAME, - encoding='UTF-8') as config_json_file: - system_conf_json = json.load(config_json_file) - - log_level_str = system_conf_json['log_level'] - log_level = logging.getLevelName(log_level_str) -except OSError: - # TODO: Print out warning that log level is incorrect or missing - log_level = _DEFAULT_LEVEL - -log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) - - -def add_file_handler(log, log_file, log_dir=_LOG_DIR): - handler = logging.FileHandler(log_dir + log_file + '.log') - handler.setFormatter(log_format) - log.addHandler(handler) - - -def add_stream_handler(log): - handler = logging.StreamHandler() - handler.setFormatter(log_format) - log.addHandler(handler) - - -def get_logger(name, log_file=None, log_dir=_LOG_DIR): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(log_level) - add_stream_handler(LOGGERS[name]) - if log_file is not None: - log_dir = log_dir if log_dir is not None else _LOG_DIR - add_file_handler(LOGGERS[name], log_file, log_dir=log_dir) - return LOGGERS[name] diff --git a/modules/test/base/python/src/util.py b/modules/test/base/python/src/util.py deleted file mode 100644 index 006648037..000000000 --- a/modules/test/base/python/src/util.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides basic utilities for a test module.""" -import subprocess -import shlex -import logger - -LOGGER = logger.get_logger('util') - -def run_command(cmd, output=True): - """Runs a process at the os level - By default, returns the standard output and error output - If the caller sets optional output parameter to False, - will only return a boolean result indicating if it was - successful in running the command. Failure is indicated - by any return code from the process other than zero.""" - - success = False - with subprocess.Popen(shlex.split(cmd), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as process: - stdout, stderr = process.communicate() - if process.returncode != 0 and output: - err_msg = f'{stderr.strip()}. Code: {process.returncode}' - LOGGER.error('Command Failed: ' + cmd) - LOGGER.error('Error: ' + err_msg) - else: - success = True - LOGGER.debug('Command succeeded: ' + cmd) - if output: - out = stdout.strip().decode('utf-8') - LOGGER.debug('Command output: ' + out) - return out, stderr - else: - return success