From d3737f4b9755fdeb1af9d92e2dd507404718d269 Mon Sep 17 00:00:00 2001 From: J Boddey Date: Tue, 27 Jun 2023 13:50:55 +0100 Subject: [PATCH 1/2] Merge dev into main (Sprint 7 and 8) (#33) * Implement test orchestrator (#4) * Initial work on test-orchestrator * Ignore runtime folder * Update runtime directory for test modules * Fix logging Add initial framework for running tests * logging and misc cleanup * logging changes * Add a stop hook after all tests complete * Refactor test_orc code * Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator * Fix baseline module Fix orchestrator exiting only after timeout * Add result file to baseline test module Change result format to match closer to design doc * Refactor pylint * Skip test module if it failed to start * Refactor * Check for valid log level --------- Co-authored-by: Jacob Boddey * Add issue report templates (#7) * Add issue templates * Update README.md * Discover devices on the network (#5) * Test run sync (#8) * Initial work on test-orchestrator * Ignore runtime folder * Update runtime directory for test modules * Fix logging Add initial framework for running tests * logging and misc cleanup * logging changes * Add a stop hook after all tests complete * Refactor test_orc code * Add arg passing Add option to use locally cloned via install or remote via main project network orchestrator * Fix baseline module Fix orchestrator exiting only after timeout * Add result file to baseline test module Change result format to match closer to design doc * Refactor pylint * Skip test module if it failed to start * Refactor * Check for valid log level * Add config file arg Misc changes to network start procedure * fix merge issues * Update runner and test orch procedure Add useful runtiem args * Restructure test run startup process Misc updates to work with net orch updates * Refactor --------- * Quick refactor (#9) * Fix duplicate sleep calls * Add net orc (#11) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add the DNS test module (#12) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add dns test module Fix test module build process * Add mac address of device under test to test container Update dns test to use mac address filter * Update dns module tests * Change result output * logging update * Update test module for better reusability * Load in module config to test module * logging cleanup * Update baseline module to new template Misc cleanup * Add ability to disable individual tests * remove duplicate readme * Update device directories * Remove local folder * Update device template Update test module to work with new device config file format * Change test module network config options Do not start network services for modules not configured for network * Refactor --------- * Add baseline and pylint tests (#25) * Discover devices on the network (#22) * Discover devices on the network * Add defaults when missing from config Implement monitor wait period from config * Add steady state monitor Remove duplicate callback registrations * Load devices into network orchestrator during testrun start --------- Co-authored-by: jhughesbiot * Build dependencies first (#21) * Build dependencies first * Remove debug message * Add depend on option to test modules * Re-add single interface option * Import subprocess --------- Co-authored-by: jhughesbiot * Port scan test module (#23) * Add network orchestrator repository * cleanup duplicate start and install scripts * Temporary fix for python dependencies * Remove duplicate python requirements * remove duplicate conf files * remove remote-net option * cleanp unecessary files * Add dns test module Fix test module build process * Add mac address of device under test to test container Update dns test to use mac address filter * Update dns module tests * Change result output * logging update * Update test module for better reusability * Load in module config to test module * logging cleanup * Update baseline module to new template Misc cleanup * Add ability to disable individual tests * remove duplicate readme * Update device directories * Remove local folder * Update device template Update test module to work with new device config file format * Change test module network config options Do not start network services for modules not configured for network * Initial nmap test module add Add device ip resolving to base module Add network mounting for test modules * Update ipv4 device resolving in test modules * Map in ip subnets and remove hard coded references * Add ftp port test * Add ability to pass config for individual tests within a module Update nmap module scan to run tests based on config * Add full module check for compliance * Add all tcp port scans to config * Update nmap commands to match existing DAQ tests Add udp scanning and tests * logging cleanup * Update TCP port scanning range Update logging * Merge device config into module config Update device template * fix merge issues * Update timeouts Add multi-threading for multiple scanns to run simultaneously Add option to use scan scripts for services * Fix merge issues * Fix device configs * Remove unecessary files * Cleanup duplicate properties * Cleanup install script * Formatting (#26) * Fix pylint issues in net orc * more pylint fixes * fix listener lint issues * fix logger lint issues * fix validator lint issues * fix util lint issues * Update base network module linting issues * Cleanup linter issues for dhcp modules Remove old code testing code * change to single quote delimeter * Cleanup linter issues for ntp module * Cleanup linter issues for radius module * Cleanup linter issues for template module * fix linter issues with faux-dev * Test results (#27) * Collect all module test results * Fix test modules without config options * Add timestamp to test results * Test results (#28) * Collect all module test results * Fix test modules without config options * Add timestamp to test results * Add attempt timing and device info to test results * Ignore disabled test containers when generating results * Fully skip modules that are disabled * Fix pylint test and skip internet tests so CI passes (#29) * disable internet checks for pass * fix pylint test * Increase pylint score (#31) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger --------- Co-authored-by: jhughesbiot * Pylint (#32) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting --------- Co-authored-by: Jacob Boddey * Add license header (#36) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module * add license header to all python files --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * Ovs (#35) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * remove ovs files added back in during merge * Nmap (#38) * More formatting fixes * More formatting fixes * More formatting fixes * More formatting fixes * Misc pylint fixes Fix test module logger * remove unused files * more formatting * revert breaking pylint changes * more formatting * fix results file * More formatting * ovs module formatting * Add ovs control into network orchestrator * Add verification methods for the base network * Add network validation and misc logging updates * remove ovs module * add license header to all python files * Update tcp scans to speed up full port range scan Add version checking Implement ssh version checking * Add unknown port checks Match unknown ports to existing services Add unknown ports without existing services to results file --------- Co-authored-by: Jacob Boddey Co-authored-by: SuperJonotron * Create startup capture (#37) * Connection (#40) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Conn mac oui (#42) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Add mac_oui test Add option to return test result and details of test for reporting * Con mac address (#43) * Initial add of connection test module with ping test * Update host user resolving * Update host user resolving for validator * add get user method to validator * Add mac_oui test Add option to return test result and details of test for reporting * Add connection.mac_address test * Dns (#44) * Add MDNS test * Update existing mdns logging to be more consistent with other tests * Add startup and monitor captures * File permissions (#45) * Fix validator file permissions * Fix test module permissions * Fix device capture file permissions * Fix device results permissions * Add connection single ip test (#47) * Nmap results (#49) * Update processing of nmap results to use xml output and json conversions for stability * Update matching with regex to prevent wrong service matches and duplicate processing for partial matches * Update max port scan range * Framework restructure (#50) * Restructure framework and modules * Fix CI paths * Fix base module * Add build script * Remove build logs * Update base and template docker files to fit the new format Implement a template option on network modules Fix skipping of base image build * remove base image build in ci * Remove group from chown --------- Co-authored-by: jhughesbiot * Ip control (#51) * Add initial work for ip control module * Implement ip control module with additional cleanup methods * Update link check to not use error stream * Add error checking around container network configurations * Add network cleanup for namespaces and links * formatting * Move config to /local (#52) * Move config to /local * Fix testing config * Fix ovs_control config location * Fix faux dev config location * Add documentation (#53) --------- Co-authored-by: jhughesbiot <50999916+jhughesbiot@users.noreply.github.com> Co-authored-by: jhughesbiot Co-authored-by: Noureddine Co-authored-by: SuperJonotron --- .gitignore | 4 +- cmd/install | 4 - cmd/start | 4 +- conf/.gitignore | 1 - docs/configure_device.md | 41 ++ docs/get_started.md | 53 +++ framework/device.py | 13 - framework/logger.py | 49 --- framework/python/src/common/logger.py | 60 +++ .../python/src/common}/util.py | 96 +++-- framework/python/src/core/device.py | 27 ++ .../{ => python/src/core}/test_runner.py | 17 +- framework/{ => python/src/core}/testrun.py | 61 +-- framework/python/src/net_orc/ip_control.py | 220 ++++++++++ .../python/src/net_orc}/listener.py | 18 +- .../python/src/net_orc/network_device.py | 24 ++ framework/python/src/net_orc/network_event.py | 23 ++ .../src/net_orc}/network_orchestrator.py | 302 +++++++------- .../python/src/net_orc}/network_validator.py | 83 +++- framework/python/src/net_orc/ovs_control.py | 186 +++++++++ .../python/src/test_orc}/module.py | 14 + .../python/src/test_orc}/runner.py | 96 +++-- .../python/src/test_orc}/test_orchestrator.py | 120 +++++- framework/requirements.txt | 9 +- local/.gitignore | 2 + {conf => local}/system.json.example | 0 .../devices/faux-dev/bin/get_default_gateway | 0 .../devices/faux-dev/bin/start_dhcp_client | 0 .../faux-dev/bin/start_network_service | 8 +- .../devices/faux-dev/conf/module_config.json | 0 .../devices/faux-dev/faux-dev.Dockerfile | 11 +- .../devices/faux-dev/python/src/dhcp_check.py | 14 + .../devices/faux-dev/python/src/dns_check.py | 14 + .../faux-dev/python/src/gateway_check.py | 14 + .../devices/faux-dev/python/src/logger.py | 14 + .../devices/faux-dev/python/src/ntp_check.py | 14 + .../devices/faux-dev/python/src/run.py | 14 + .../devices/faux-dev/python/src/util.py | 14 + .../network}/base/base.Dockerfile | 9 +- .../network}/base/bin/capture | 2 +- .../network}/base/bin/setup_binaries | 0 .../network}/base/bin/start_grpc | 0 .../network}/base/bin/start_module | 0 .../network}/base/bin/start_network_service | 0 .../network}/base/bin/wait_for_interface | 0 .../network}/base/conf/module_config.json | 0 .../network}/base/python/requirements.txt | 0 .../base/python/src/grpc/start_server.py | 14 + .../network}/base/python/src/logger.py | 14 + .../network}/dhcp-1/bin/start_network_service | 4 +- .../network}/dhcp-1/conf/dhcpd.conf | 0 .../network}/dhcp-1/conf/module_config.json | 0 .../network}/dhcp-1/conf/radvd.conf | 0 .../network/dhcp-1/dhcp-1.Dockerfile | 9 +- .../dhcp-1/python/src/grpc/__init__.py | 0 .../dhcp-1/python/src/grpc/dhcp_config.py | 14 + .../dhcp-1/python/src/grpc/network_service.py | 14 + .../dhcp-1/python/src/grpc/proto/grpc.proto | 0 .../network}/dhcp-2/bin/start_network_service | 4 +- .../network}/dhcp-2/conf/dhcpd.conf | 0 .../network}/dhcp-2/conf/module_config.json | 0 .../network}/dhcp-2/conf/radvd.conf | 0 .../network/dhcp-2/dhcp-2.Dockerfile | 12 +- .../dhcp-2/python/src/grpc/__init__.py | 0 .../dhcp-2/python/src/grpc/dhcp_config.py | 14 + .../dhcp-2/python/src/grpc/network_service.py | 14 + .../dhcp-2/python/src/grpc/proto/grpc.proto | 0 .../network}/dns/bin/start_network_service | 0 .../network}/dns/conf/dnsmasq.conf | 0 .../network}/dns/conf/module_config.json | 0 .../network}/dns/dns.Dockerfile | 7 +- .../gateway/bin/start_network_service | 0 .../network}/gateway/conf/module_config.json | 0 .../network}/gateway/gateway.Dockerfile | 7 +- .../network}/ntp/bin/start_network_service | 2 +- .../network}/ntp/conf/module_config.json | 0 modules/network/ntp/ntp.Dockerfile | 16 + .../network}/ntp/python/src/ntp_server.py | 14 + .../network}/radius/bin/start_network_service | 2 +- .../network}/radius/conf/ca.crt | 0 .../network}/radius/conf/eap | 0 .../network}/radius/conf/module_config.json | 0 .../network}/radius/python/requirements.txt | 0 .../radius/python/src/authenticator.py | 14 + .../network}/radius/radius.Dockerfile | 9 +- .../template/bin/start_network_service | 0 .../network}/template/conf/module_config.json | 1 + .../template/python/src/template_main.py | 18 + modules/network/template/template.Dockerfile | 14 + .../test}/base/base.Dockerfile | 9 +- .../modules => modules/test}/base/bin/capture | 2 +- .../test}/base/bin/get_ipv4_addr | 0 .../test}/base/bin/setup_binaries | 0 .../test}/base/bin/start_grpc | 0 .../test}/base/bin/start_module | 6 + .../test}/base/bin/wait_for_interface | 0 .../test}/base/conf/module_config.json | 0 .../test}/base/python/requirements.txt | 0 .../base/python/src/grpc/start_server.py | 14 + .../test}/base/python/src/logger.py | 14 + .../test}/base/python/src/test_module.py | 21 +- .../test}/base/python/src/util.py | 14 + modules/test/baseline/baseline.Dockerfile | 14 + .../test}/baseline/bin/start_test_module | 4 +- .../test}/baseline/conf/module_config.json | 0 .../baseline/python/src/baseline_module.py | 14 + .../test}/baseline/python/src/run.py | 14 + modules/test/conn/bin/start_test_module | 39 ++ modules/test/conn/conf/module_config.json | 37 ++ modules/test/conn/conn.Dockerfile | 26 ++ modules/test/conn/python/requirements.txt | 1 + .../test/conn/python/src/connection_module.py | 117 ++++++ modules/test/conn/python/src/run.py | 68 ++++ .../test}/dns/bin/start_test_module | 4 +- .../test}/dns/conf/module_config.json | 4 + modules/test/dns/dns.Dockerfile | 14 + .../test}/dns/python/src/dns_module.py | 51 ++- .../test}/dns/python/src/run.py | 22 +- .../test}/nmap/bin/start_test_module | 4 +- .../test}/nmap/conf/module_config.json | 3 +- modules/test/nmap/nmap.Dockerfile | 20 + modules/test/nmap/python/requirements.txt | 1 + modules/test/nmap/python/src/nmap_module.py | 381 ++++++++++++++++++ .../test}/nmap/python/src/run.py | 27 +- net_orc/.gitignore | 133 ------ net_orc/docker-compose.yml | 64 --- net_orc/network/modules/ntp/ntp.Dockerfile | 13 - .../modules/ovs/bin/start_network_service | 22 - .../modules/ovs/conf/module_config.json | 24 -- net_orc/network/modules/ovs/ovs.Dockerfile | 20 - .../modules/ovs/python/requirements.txt | 0 .../network/modules/ovs/python/src/logger.py | 16 - .../modules/ovs/python/src/ovs_control.py | 105 ----- net_orc/network/modules/ovs/python/src/run.py | 54 --- .../network/modules/ovs/python/src/util.py | 23 -- .../template/python/src/template_main.py | 4 - .../modules/template/template.Dockerfile | 11 - net_orc/orchestrator.Dockerfile | 22 - net_orc/python/requirements.txt | 4 - net_orc/python/src/network_device.py | 10 - net_orc/python/src/network_event.py | 9 - .../{Template => template}/device_config.json | 0 test_orc/modules/baseline/baseline.Dockerfile | 11 - test_orc/modules/dns/dns.Dockerfile | 11 - test_orc/modules/nmap/nmap.Dockerfile | 11 - .../modules/nmap/python/src/nmap_module.py | 227 ----------- test_orc/python/requirements.txt | 0 testing/test_baseline | 7 +- testing/test_baseline.py | 14 + 149 files changed, 2424 insertions(+), 1208 deletions(-) delete mode 100644 conf/.gitignore create mode 100644 docs/configure_device.md create mode 100644 docs/get_started.md delete mode 100644 framework/device.py delete mode 100644 framework/logger.py create mode 100644 framework/python/src/common/logger.py rename {net_orc/python/src => framework/python/src/common}/util.py (64%) create mode 100644 framework/python/src/core/device.py rename framework/{ => python/src/core}/test_runner.py (79%) rename framework/{ => python/src/core}/testrun.py (76%) create mode 100644 framework/python/src/net_orc/ip_control.py rename {net_orc/python/src => framework/python/src/net_orc}/listener.py (77%) create mode 100644 framework/python/src/net_orc/network_device.py create mode 100644 framework/python/src/net_orc/network_event.py rename {net_orc/python/src => framework/python/src/net_orc}/network_orchestrator.py (74%) rename {net_orc/python/src => framework/python/src/net_orc}/network_validator.py (74%) create mode 100644 framework/python/src/net_orc/ovs_control.py rename {test_orc/python/src => framework/python/src/test_orc}/module.py (50%) rename {test_orc/python/src => framework/python/src/test_orc}/runner.py (65%) rename {test_orc/python/src => framework/python/src/test_orc}/test_orchestrator.py (68%) create mode 100644 local/.gitignore rename {conf => local}/system.json.example (100%) rename {net_orc/network => modules}/devices/faux-dev/bin/get_default_gateway (100%) rename {net_orc/network => modules}/devices/faux-dev/bin/start_dhcp_client (100%) rename {net_orc/network => modules}/devices/faux-dev/bin/start_network_service (79%) rename {net_orc/network => modules}/devices/faux-dev/conf/module_config.json (100%) rename {net_orc/network => modules}/devices/faux-dev/faux-dev.Dockerfile (65%) rename {net_orc/network => modules}/devices/faux-dev/python/src/dhcp_check.py (82%) rename {net_orc/network => modules}/devices/faux-dev/python/src/dns_check.py (82%) rename {net_orc/network => modules}/devices/faux-dev/python/src/gateway_check.py (65%) rename {net_orc/network => modules}/devices/faux-dev/python/src/logger.py (67%) rename {net_orc/network => modules}/devices/faux-dev/python/src/ntp_check.py (78%) rename {net_orc/network => modules}/devices/faux-dev/python/src/run.py (84%) rename {net_orc/network => modules}/devices/faux-dev/python/src/util.py (60%) rename {net_orc/network/modules => modules/network}/base/base.Dockerfile (74%) rename {net_orc/network/modules => modules/network}/base/bin/capture (90%) rename {net_orc/network/modules => modules/network}/base/bin/setup_binaries (100%) rename {net_orc/network/modules => modules/network}/base/bin/start_grpc (100%) rename {net_orc/network/modules => modules/network}/base/bin/start_module (100%) rename {net_orc/network/modules => modules/network}/base/bin/start_network_service (100%) rename {net_orc/network/modules => modules/network}/base/bin/wait_for_interface (100%) rename {net_orc/network/modules => modules/network}/base/conf/module_config.json (100%) rename {net_orc/network/modules => modules/network}/base/python/requirements.txt (100%) rename {net_orc/network/modules => modules/network}/base/python/src/grpc/start_server.py (60%) rename {net_orc/network/modules => modules/network}/base/python/src/logger.py (68%) rename {net_orc/network/modules => modules/network}/dhcp-1/bin/start_network_service (91%) rename {net_orc/network/modules => modules/network}/dhcp-1/conf/dhcpd.conf (100%) rename {net_orc/network/modules => modules/network}/dhcp-1/conf/module_config.json (100%) rename {net_orc/network/modules => modules/network}/dhcp-1/conf/radvd.conf (100%) rename net_orc/network/modules/dhcp-2/dhcp-2.Dockerfile => modules/network/dhcp-1/dhcp-1.Dockerfile (56%) rename {net_orc/network/modules => modules/network}/dhcp-1/python/src/grpc/__init__.py (100%) rename {net_orc/network/modules => modules/network}/dhcp-1/python/src/grpc/dhcp_config.py (91%) rename {net_orc/network/modules => modules/network}/dhcp-1/python/src/grpc/network_service.py (70%) rename {net_orc/network/modules => modules/network}/dhcp-1/python/src/grpc/proto/grpc.proto (100%) rename {net_orc/network/modules => modules/network}/dhcp-2/bin/start_network_service (91%) rename {net_orc/network/modules => modules/network}/dhcp-2/conf/dhcpd.conf (100%) rename {net_orc/network/modules => modules/network}/dhcp-2/conf/module_config.json (100%) rename {net_orc/network/modules => modules/network}/dhcp-2/conf/radvd.conf (100%) rename net_orc/network/modules/dhcp-1/dhcp-1.Dockerfile => modules/network/dhcp-2/dhcp-2.Dockerfile (55%) rename {net_orc/network/modules => modules/network}/dhcp-2/python/src/grpc/__init__.py (100%) rename {net_orc/network/modules => modules/network}/dhcp-2/python/src/grpc/dhcp_config.py (91%) rename {net_orc/network/modules => modules/network}/dhcp-2/python/src/grpc/network_service.py (70%) rename {net_orc/network/modules => modules/network}/dhcp-2/python/src/grpc/proto/grpc.proto (100%) rename {net_orc/network/modules => modules/network}/dns/bin/start_network_service (100%) rename {net_orc/network/modules => modules/network}/dns/conf/dnsmasq.conf (100%) rename {net_orc/network/modules => modules/network}/dns/conf/module_config.json (100%) rename {net_orc/network/modules => modules/network}/dns/dns.Dockerfile (67%) rename {net_orc/network/modules => modules/network}/gateway/bin/start_network_service (100%) rename {net_orc/network/modules => modules/network}/gateway/conf/module_config.json (100%) rename {net_orc/network/modules => modules/network}/gateway/gateway.Dockerfile (59%) rename {net_orc/network/modules => modules/network}/ntp/bin/start_network_service (82%) rename {net_orc/network/modules => modules/network}/ntp/conf/module_config.json (100%) create mode 100644 modules/network/ntp/ntp.Dockerfile rename {net_orc/network/modules => modules/network}/ntp/python/src/ntp_server.py (90%) rename {net_orc/network/modules => modules/network}/radius/bin/start_network_service (89%) rename {net_orc/network/modules => modules/network}/radius/conf/ca.crt (100%) rename {net_orc/network/modules => modules/network}/radius/conf/eap (100%) rename {net_orc/network/modules => modules/network}/radius/conf/module_config.json (100%) rename {net_orc/network/modules => modules/network}/radius/python/requirements.txt (100%) rename {net_orc/network/modules => modules/network}/radius/python/src/authenticator.py (71%) rename {net_orc/network/modules => modules/network}/radius/radius.Dockerfile (74%) rename {net_orc/network/modules => modules/network}/template/bin/start_network_service (100%) rename {net_orc/network/modules => modules/network}/template/conf/module_config.json (91%) create mode 100644 modules/network/template/python/src/template_main.py create mode 100644 modules/network/template/template.Dockerfile rename {test_orc/modules => modules/test}/base/base.Dockerfile (74%) rename {test_orc/modules => modules/test}/base/bin/capture (88%) rename {test_orc/modules => modules/test}/base/bin/get_ipv4_addr (100%) rename {test_orc/modules => modules/test}/base/bin/setup_binaries (100%) rename {test_orc/modules => modules/test}/base/bin/start_grpc (100%) rename {test_orc/modules => modules/test}/base/bin/start_module (91%) rename {test_orc/modules => modules/test}/base/bin/wait_for_interface (100%) rename {test_orc/modules => modules/test}/base/conf/module_config.json (100%) rename {test_orc/modules => modules/test}/base/python/requirements.txt (100%) rename {test_orc/modules => modules/test}/base/python/src/grpc/start_server.py (60%) rename {test_orc/modules => modules/test}/base/python/src/logger.py (67%) rename {test_orc/modules => modules/test}/base/python/src/test_module.py (80%) rename {test_orc/modules => modules/test}/base/python/src/util.py (61%) create mode 100644 modules/test/baseline/baseline.Dockerfile rename {test_orc/modules => modules/test}/baseline/bin/start_test_module (90%) rename {test_orc/modules => modules/test}/baseline/conf/module_config.json (100%) rename {test_orc/modules => modules/test}/baseline/python/src/baseline_module.py (53%) rename {test_orc/modules => modules/test}/baseline/python/src/run.py (69%) create mode 100644 modules/test/conn/bin/start_test_module create mode 100644 modules/test/conn/conf/module_config.json create mode 100644 modules/test/conn/conn.Dockerfile create mode 100644 modules/test/conn/python/requirements.txt create mode 100644 modules/test/conn/python/src/connection_module.py create mode 100644 modules/test/conn/python/src/run.py rename {test_orc/modules => modules/test}/dns/bin/start_test_module (90%) rename {test_orc/modules => modules/test}/dns/conf/module_config.json (80%) create mode 100644 modules/test/dns/dns.Dockerfile rename {test_orc/modules => modules/test}/dns/python/src/dns_module.py (50%) rename {test_orc/modules => modules/test}/dns/python/src/run.py (64%) rename {test_orc/modules => modules/test}/nmap/bin/start_test_module (93%) rename {test_orc/modules => modules/test}/nmap/conf/module_config.json (96%) create mode 100644 modules/test/nmap/nmap.Dockerfile create mode 100644 modules/test/nmap/python/requirements.txt create mode 100644 modules/test/nmap/python/src/nmap_module.py rename {test_orc/modules => modules/test}/nmap/python/src/run.py (58%) delete mode 100644 net_orc/.gitignore delete mode 100644 net_orc/docker-compose.yml delete mode 100644 net_orc/network/modules/ntp/ntp.Dockerfile delete mode 100644 net_orc/network/modules/ovs/bin/start_network_service delete mode 100644 net_orc/network/modules/ovs/conf/module_config.json delete mode 100644 net_orc/network/modules/ovs/ovs.Dockerfile delete mode 100644 net_orc/network/modules/ovs/python/requirements.txt delete mode 100644 net_orc/network/modules/ovs/python/src/logger.py delete mode 100644 net_orc/network/modules/ovs/python/src/ovs_control.py delete mode 100644 net_orc/network/modules/ovs/python/src/run.py delete mode 100644 net_orc/network/modules/ovs/python/src/util.py delete mode 100644 net_orc/network/modules/template/python/src/template_main.py delete mode 100644 net_orc/network/modules/template/template.Dockerfile delete mode 100644 net_orc/orchestrator.Dockerfile delete mode 100644 net_orc/python/requirements.txt delete mode 100644 net_orc/python/src/network_device.py delete mode 100644 net_orc/python/src/network_event.py rename resources/devices/{Template => template}/device_config.json (100%) delete mode 100644 test_orc/modules/baseline/baseline.Dockerfile delete mode 100644 test_orc/modules/dns/dns.Dockerfile delete mode 100644 test_orc/modules/nmap/nmap.Dockerfile delete mode 100644 test_orc/modules/nmap/python/src/nmap_module.py delete mode 100644 test_orc/python/requirements.txt diff --git a/.gitignore b/.gitignore index 5dfc1f6f9..e168ec07a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ venv/ .vscode/ error pylint.out -local/ -__pycache__/ \ No newline at end of file +__pycache__/ +build/ \ No newline at end of file diff --git a/cmd/install b/cmd/install index 23e463158..37c03e113 100755 --- a/cmd/install +++ b/cmd/install @@ -6,8 +6,4 @@ source venv/bin/activate pip3 install -r framework/requirements.txt -pip3 install -r net_orc/python/requirements.txt - -pip3 install -r test_orc/python/requirements.txt - deactivate diff --git a/cmd/start b/cmd/start index d146f413d..55d2e52eb 100755 --- a/cmd/start +++ b/cmd/start @@ -18,7 +18,9 @@ rm -rf runtime source venv/bin/activate # TODO: Execute python code -python -u framework/test_runner.py $@ +# Set the PYTHONPATH to include the "src" directory +export PYTHONPATH="$PWD/framework/python/src" +python -u framework/python/src/core/test_runner.py $@ # TODO: Work in progress code for containerization of OVS module # asyncRun() { diff --git a/conf/.gitignore b/conf/.gitignore deleted file mode 100644 index 41b89ceb1..000000000 --- a/conf/.gitignore +++ /dev/null @@ -1 +0,0 @@ -system.json \ No newline at end of file diff --git a/docs/configure_device.md b/docs/configure_device.md new file mode 100644 index 000000000..ad58521a4 --- /dev/null +++ b/docs/configure_device.md @@ -0,0 +1,41 @@ +# Device Configuration + +The device configuration file allows you to customize the testing behavior for a specific device. This file is located at `local/devices/{Device Name}/device_config.json`. Below is an overview of how to configure the device tests. + +## Device Information + +The device information section includes the manufacturer, model, and MAC address of the device. These details help identify the specific device being tested. + +## Test Modules + +Test modules are groups of tests that can be enabled or disabled as needed. You can choose which test modules to include for your device. The device configuration file contains the following test module: + +- DNS Test Module + +### Enabling and Disabling Test Modules + +To enable or disable a test module, modify the `enabled` field within the respective module. Setting it to `true` enables the module, while setting it to `false` disables the module. + +## Individual Tests + +Within the DNS test module, there are individual tests that can be enabled or disabled. These tests focus on specific aspects of network behavior. You can customize the tests based on your device and testing requirements. + +### Enabling and Disabling Tests + +To enable or disable an individual test, modify the `enabled` field within the respective test. Setting it to `true` enables the test, while setting it to `false` disables the test. + +> Note: The example device configuration file (`resources/devices/template/device_config.json`) provides a complete usage example, including the structure and configuration options for the DNS test module and its tests. You can refer to this file to understand how to configure your device tests effectively. + +## Customizing the Device Configuration + +To customize the device configuration for your specific device, follow these steps: + +1. Copy the default configuration file provided in the `resources/devices/template` folder. + - Create a new folder for your device under `local/devices` directory. + - Copy the `device_config.json` file from `resources/devices/template` to the newly created device folder. + +This ensures that you have a copy of the default configuration file, which you can then modify for your specific device. + +> Note: Ensure that the device configuration file is properly formatted, and the changes made align with the intended test behavior. Incorrect settings or syntax may lead to unexpected results during testing. + +If you encounter any issues or need assistance with the device configuration, refer to the Test Run documentation or ask a question on the Issues page. diff --git a/docs/get_started.md b/docs/get_started.md new file mode 100644 index 000000000..7b8cf9e13 --- /dev/null +++ b/docs/get_started.md @@ -0,0 +1,53 @@ +# Getting Started + +## Prerequisites + +### Hardware + +Before starting with Test Run, ensure you have the following hardware: + +- PC running Ubuntu LTS (laptop or desktop) +- 2x USB Ethernet adapter (one may be a built-in Ethernet port) +- Internet connection + +### Software + +Ensure the following software is installed on your Ubuntu LTS PC: + +- Python 3 (already available on Ubuntu LTS) +- Docker - Installation Guide: [https://docs.docker.com/engine/install/](https://docs.docker.com/engine/install/) +- Open vSwitch ``sudo apt-get install openvswitch-common openvswitch-switch`` + +## Installation + +1. Download Test Run from the releases page or the appropriate source. + +2. Run the install script. + +## Configuration + +1. Copy the default configuration file. + +2. Open the `local/system.json` file and modify the configuration as needed. Specify the interface names for the internet and device interfaces. + +## Test Your Device + +1. Attach network interfaces: + + - Connect one USB Ethernet adapter to the internet source (e.g., router or switch) using an Ethernet cable. + - Connect the other USB Ethernet adapter directly to the IoT device you want to test using an Ethernet cable. + +2. Start Test Run. + + - To run Test Run in network-only mode (without running any tests), use the `--net-only` option. + + - To skip network validation before use and not launch the faux device on startup, use the `--no-validate` option. + +# Troubleshooting + +If you encounter any issues or need assistance, consider the following: + +- Ensure that all hardware and software prerequisites are met. +- Verify that the network interfaces are connected correctly. +- Check the configuration in the `local/system.json` file. +- Refer to the Test Run documentation or ask for further assistance from the support team. diff --git a/framework/device.py b/framework/device.py deleted file mode 100644 index eef275d54..000000000 --- a/framework/device.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Track device object information.""" - -from network_device import NetworkDevice -from dataclasses import dataclass - - -@dataclass -class Device(NetworkDevice): - """Represents a physical device and it's configuration.""" - - make: str = None - model: str = None - test_modules: str = None diff --git a/framework/logger.py b/framework/logger.py deleted file mode 100644 index d4702cb38..000000000 --- a/framework/logger.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Manages stream and file loggers.""" -import json -import logging -import os - -LOGGERS = {} -_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' -_DATE_FORMAT = '%b %02d %H:%M:%S' -_DEFAULT_LOG_LEVEL = logging.INFO -_LOG_LEVEL = logging.INFO -_CONF_DIR = 'conf' -_CONF_FILE_NAME = 'system.json' -_LOG_DIR = 'runtime/testing/' - -# Set log level -with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), - encoding='utf-8') as system_conf_file: - system_conf_json = json.load(system_conf_file) -log_level_str = system_conf_json['log_level'] - -temp_log = logging.getLogger('temp') -try: - temp_log.setLevel(logging.getLevelName(log_level_str)) - _LOG_LEVEL = logging.getLevelName(log_level_str) -except ValueError: - print('Invalid log level set in ' + _CONF_DIR + '/' + _CONF_FILE_NAME + - '. Using INFO as log level') - _LOG_LEVEL = _DEFAULT_LOG_LEVEL - -log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) - -def add_file_handler(log, log_file): - handler = logging.FileHandler(_LOG_DIR + log_file + '.log') - handler.setFormatter(log_format) - log.addHandler(handler) - -def add_stream_handler(log): - handler = logging.StreamHandler() - handler.setFormatter(log_format) - log.addHandler(handler) - -def get_logger(name, log_file=None): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(_LOG_LEVEL) - add_stream_handler(LOGGERS[name]) - if log_file is not None: - add_file_handler(LOGGERS[name], log_file) - return LOGGERS[name] diff --git a/framework/python/src/common/logger.py b/framework/python/src/common/logger.py new file mode 100644 index 000000000..539767f53 --- /dev/null +++ b/framework/python/src/common/logger.py @@ -0,0 +1,60 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sets up the logger to be used for the test modules.""" +import json +import logging +import os + +LOGGERS = {} +_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' +_DATE_FORMAT = '%b %02d %H:%M:%S' +_DEFAULT_LEVEL = logging.INFO +_CONF_DIR = 'conf' +_CONF_FILE_NAME = 'system.json' + +# Set log level +try: + with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), + encoding='UTF-8') as config_json_file: + system_conf_json = json.load(config_json_file) + + log_level_str = system_conf_json['log_level'] + log_level = logging.getLevelName(log_level_str) +except OSError: + # TODO: Print out warning that log level is incorrect or missing + log_level = _DEFAULT_LEVEL + +log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) + +def add_file_handler(log, log_file, log_dir): + handler = logging.FileHandler(log_dir + log_file + '.log') + handler.setFormatter(log_format) + log.addHandler(handler) + + +def add_stream_handler(log): + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) + + +def get_logger(name, log_file=None, log_dir=None): + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(log_level) + add_stream_handler(LOGGERS[name]) + if log_file is not None and log_dir is not None: + add_file_handler(LOGGERS[name], log_file, log_dir) + return LOGGERS[name] diff --git a/net_orc/python/src/util.py b/framework/python/src/common/util.py similarity index 64% rename from net_orc/python/src/util.py rename to framework/python/src/common/util.py index a7b07ddf9..1ffe70651 100644 --- a/net_orc/python/src/util.py +++ b/framework/python/src/common/util.py @@ -1,41 +1,55 @@ -"""Provides basic utilities for the network orchestrator.""" -import subprocess -import shlex -import logger -import netifaces - -LOGGER = logger.get_logger('util') - - -def run_command(cmd, output=True): - """Runs a process at the os level - By default, returns the standard output and error output - If the caller sets optional output parameter to False, - will only return a boolean result indicating if it was - succesful in running the command. Failure is indicated - by any return code from the process other than zero.""" - - success = False - process = subprocess.Popen(shlex.split(cmd), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - - if process.returncode != 0 and output: - err_msg = f'{stderr.strip()}. Code: {process.returncode}' - LOGGER.error('Command Failed: ' + cmd) - LOGGER.error('Error: ' + err_msg) - else: - success = True - if output: - return stdout.strip().decode('utf-8'), stderr - else: - return success - - -def interface_exists(interface): - return interface in netifaces.interfaces() - - -def prettify(mac_string): - return ':'.join([f'{ord(b):02x}' for b in mac_string]) +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides basic utilities for the network orchestrator.""" +import subprocess +import shlex +from common import logger +import netifaces + +LOGGER = logger.get_logger('util') + + +def run_command(cmd, output=True): + """Runs a process at the os level + By default, returns the standard output and error output + If the caller sets optional output parameter to False, + will only return a boolean result indicating if it was + succesful in running the command. Failure is indicated + by any return code from the process other than zero.""" + + success = False + process = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + + if process.returncode != 0 and output: + err_msg = f'{stderr.strip()}. Code: {process.returncode}' + LOGGER.error('Command Failed: ' + cmd) + LOGGER.error('Error: ' + err_msg) + else: + success = True + if output: + return stdout.strip().decode('utf-8'), stderr + else: + return success + + +def interface_exists(interface): + return interface in netifaces.interfaces() + + +def prettify(mac_string): + return ':'.join([f'{ord(b):02x}' for b in mac_string]) diff --git a/framework/python/src/core/device.py b/framework/python/src/core/device.py new file mode 100644 index 000000000..44f275bdf --- /dev/null +++ b/framework/python/src/core/device.py @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Track device object information.""" + +from net_orc.network_device import NetworkDevice +from dataclasses import dataclass + + +@dataclass +class Device(NetworkDevice): + """Represents a physical device and it's configuration.""" + + make: str = None + model: str = None + test_modules: str = None diff --git a/framework/test_runner.py b/framework/python/src/core/test_runner.py similarity index 79% rename from framework/test_runner.py rename to framework/python/src/core/test_runner.py index 0733d4353..226f874cc 100644 --- a/framework/test_runner.py +++ b/framework/python/src/core/test_runner.py @@ -1,4 +1,17 @@ -#!/usr/bin/env python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Wrapper for the TestRun that simplifies virtual testing procedure by allowing direct calling from the command line. @@ -10,7 +23,7 @@ import argparse import sys from testrun import TestRun -import logger +from common import logger import signal LOGGER = logger.get_logger("runner") diff --git a/framework/testrun.py b/framework/python/src/core/testrun.py similarity index 76% rename from framework/testrun.py rename to framework/python/src/core/testrun.py index 94ad2ef9f..d613410e9 100644 --- a/framework/testrun.py +++ b/framework/python/src/core/testrun.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """The overall control of the Test Run application. This file provides the integration between all of the @@ -11,30 +25,23 @@ import json import signal import time -import logger +from common import logger # Locate parent directory current_dir = os.path.dirname(os.path.realpath(__file__)) -parent_dir = os.path.dirname(current_dir) - -# Add net_orc to Python path -net_orc_dir = os.path.join(parent_dir, 'net_orc', 'python', 'src') -sys.path.append(net_orc_dir) - -# Add test_orc to Python path -test_orc_dir = os.path.join(parent_dir, 'test_orc', 'python', 'src') -sys.path.append(test_orc_dir) -from listener import NetworkEvent # pylint: disable=wrong-import-position,import-outside-toplevel -import test_orchestrator as test_orc # pylint: disable=wrong-import-position,import-outside-toplevel -import network_orchestrator as net_orc # pylint: disable=wrong-import-position,import-outside-toplevel +# Locate the test-run root directory, 4 levels, src->python->framework->test-run +root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) -from device import Device # pylint: disable=wrong-import-position,import-outside-toplevel +from net_orc.listener import NetworkEvent +from test_orc import test_orchestrator as test_orc +from net_orc import network_orchestrator as net_orc +from device import Device LOGGER = logger.get_logger('test_run') -CONFIG_FILE = 'conf/system.json' -EXAMPLE_CONFIG_FILE = 'conf/system.json.example' -RUNTIME = 1500 +CONFIG_FILE = 'local/system.json' +EXAMPLE_CONFIG_FILE = 'local/system.json.example' +RUNTIME = 120 LOCAL_DEVICES_DIR = 'local/devices' RESOURCE_DEVICES_DIR = 'resources/devices' @@ -44,7 +51,6 @@ DEVICE_MAC_ADDR = 'mac_addr' DEVICE_TEST_MODULES = 'test_modules' - class TestRun: # pylint: disable=too-few-public-methods """Test Run controller. @@ -95,12 +101,17 @@ def start(self): [NetworkEvent.DEVICE_DISCOVERED] ) + self._net_orc.start_listener() LOGGER.info('Waiting for devices on the network...') - # Check timeout and whether testing is currently - # in progress before stopping time.sleep(RUNTIME) + if not self._test_orc.test_in_progress(): + LOGGER.info('Timed out whilst waiting for device') + else: + while self._test_orc.test_in_progress(): + time.sleep(5) + self.stop() def stop(self, kill=False): @@ -123,7 +134,7 @@ def _exit_handler(self, signum, arg): # pylint: disable=unused-argument def _get_config_abs(self, config_file=None): if config_file is None: # If not defined, use relative pathing to local file - config_file = os.path.join(parent_dir, CONFIG_FILE) + config_file = os.path.join(root_dir, CONFIG_FILE) # Expand the config file to absolute pathing return os.path.abspath(config_file) @@ -132,14 +143,6 @@ def _start_network(self): # Start the network orchestrator self._net_orc.start() - def _run_tests(self, device): - """Iterate through and start all test modules.""" - - # To Do: Make this configurable - time.sleep(60) # Let device bootup - - self._test_orc.run_test_modules(device) - def _stop_network(self, kill=False): self._net_orc.stop(kill=kill) diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py new file mode 100644 index 000000000..eb683c46b --- /dev/null +++ b/framework/python/src/net_orc/ip_control.py @@ -0,0 +1,220 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""IP Control Module""" +from common import logger +from common import util +import re + +LOGGER = logger.get_logger('ip_ctrl') + + +class IPControl: + """IP Control""" + + def __init__(self): + """Initialize the IPControl object""" + + def add_link(self, interface_name, peer_name): + """Create an ip link with a peer""" + success = util.run_command('ip link add ' + interface_name + + ' type veth peer name ' + peer_name) + return success + + def add_namespace(self, namespace): + """Add a network namespace""" + exists = self.namespace_exists(namespace) + LOGGER.info("Namespace exists: " + str(exists)) + if exists: + return True + else: + success = util.run_command('ip netns add ' + namespace) + return success + + def delete_link(self, interface_name): + """Delete an ip link""" + success = util.run_command('ip link delete ' + interface_name) + return success + + def delete_namespace(self, interface_name): + """Delete an ip namespace""" + success = util.run_command('ip netns delete ' + interface_name) + return success + + def link_exists(self, link_name): + links = self.get_links() + return link_name in links + + def namespace_exists(self, namespace): + """Check if a namespace already exists""" + namespaces = self.get_namespaces() + if namespace in namespaces: + return True + else: + return False + + def get_links(self): + stdout, stderr = util.run_command('ip link list') + links = stdout.strip().split('\n') + netns_links = [] + for link in links: + match = re.search(r'\d+:\s+(\S+)', link) + if match: + interface_name = match.group(1) + name_match = re.search(r'(.*)@', interface_name) + if name_match: + interface_name = name_match.group(1) + netns_links.append(interface_name.strip()) + return netns_links + + def get_namespaces(self): + stdout, stderr = util.run_command('ip netns list') + #Strip ID's from the namespace results + namespaces = re.findall(r'(\S+)(?:\s+\(id: \d+\))?', stdout) + return namespaces + + def set_namespace(self, interface_name, namespace): + """Attach an interface to a network namespace""" + success = util.run_command('ip link set ' + interface_name + ' netns ' + + namespace) + return success + + def rename_interface(self, interface_name, namespace, new_name): + """Rename an interface""" + success = util.run_command('ip netns exec ' + namespace + + ' ip link set dev ' + interface_name + ' name ' + + new_name) + return success + + def set_interface_mac(self, interface_name, namespace, mac_addr): + """Set MAC address of an interface""" + success = util.run_command('ip netns exec ' + namespace + + ' ip link set dev ' + interface_name + + ' address ' + mac_addr) + return success + + def set_interface_ip(self, interface_name, namespace, ipaddr): + """Set IP address of an interface""" + success = util.run_command('ip netns exec ' + namespace + ' ip addr add ' + + ipaddr + ' dev ' + interface_name) + return success + + def set_interface_up(self, interface_name, namespace=None): + """Set the interface to the up state""" + if namespace is None: + success = util.run_command('ip link set dev ' + interface_name + ' up') + else: + success = util.run_command('ip netns exec ' + namespace + + ' ip link set dev ' + interface_name + ' up') + return success + + def clean_all(self): + """Cleanup all existing test run interfaces and namespaces""" + + # Delete all namesapces that start with tr + namespaces = self.get_namespaces() + for ns in namespaces: + if 'tr' in ns: + self.delete_namespace(ns) + + # Delete all namespaces that start with tr + links = self.get_links() + for link in links: + if 'tr' in link: + self.delete_link(link) + + def cleanup(self, interface=None, namespace=None): + """Cleanup existing link and namespace if they still exist""" + + link_clean = True + if interface is not None: + if self.link_exists(interface): + link_clean = self.delete_link(interface) + + ns_clean = True + if namespace is not None: + if self.namespace_exists(namespace): + ns_clean = self.delete_namespace + return link_clean and ns_clean + + def configure_container_interface(self, + bridge_intf, + container_intf, + namespace_intf, + namespace, + mac_addr, + container_name=None, + ipv4_addr=None, + ipv6_addr=None): + + # Cleanup old interface and namespaces + self.cleanup(bridge_intf, namespace) + + # Create interface pair + self.add_link(bridge_intf, container_intf) + + if container_name is not None: + # Get PID for running container + # TODO: Some error checking around missing PIDs might be required + container_pid = util.run_command('docker inspect -f {{.State.Pid}} ' + + container_name)[0] + if not container_pid.isdigit(): + LOGGER.error(f'Failed to resolve pid for {container_name}') + return False + + # Create symlink for container network namespace + if not util.run_command('ln -sf /proc/' + container_pid + + '/ns/net /var/run/netns/' + namespace, + output=False): + LOGGER.error( + f'Failed to link {container_name} to namespace {namespace_intf}') + return False + + # Attach container interface to container network namespace + if not self.set_namespace(container_intf, namespace): + LOGGER.error(f'Failed to set namespace {namespace} for {container_intf}') + return False + + # Rename container interface name + if not self.rename_interface(container_intf, namespace, namespace_intf): + LOGGER.error( + f'Failed to rename container interface {container_intf} to {namespace_intf}' + ) + return False + + # Set MAC address of container interface + if not self.set_interface_mac(namespace_intf, namespace, mac_addr): + LOGGER.error( + f'Failed to set MAC address for {namespace_intf} to {mac_addr}') + return False + + # Set IP address of container interface + if ipv4_addr is not None: + if not self.set_interface_ip(namespace_intf, namespace, ipv4_addr): + LOGGER.error( + f'Failed to set IPv4 address for {namespace_intf} to {ipv4_addr}') + return False + if ipv6_addr is not None: + if not self.set_interface_ip(namespace_intf, namespace, ipv6_addr): + LOGGER.error( + f'Failed to set IPv6 address for {namespace_intf} to {ipv6_addr}') + return False + + # Set interfaces up + if not self.set_interface_up(bridge_intf): + LOGGER.error(f'Failed to set interface up {bridge_intf}') + return False + if not self.set_interface_up(namespace_intf, namespace): + LOGGER.error(f'Failed to set interface up {namespace_intf}') + return False + return True diff --git a/net_orc/python/src/listener.py b/framework/python/src/net_orc/listener.py similarity index 77% rename from net_orc/python/src/listener.py rename to framework/python/src/net_orc/listener.py index de7a07616..4f8e1961f 100644 --- a/net_orc/python/src/listener.py +++ b/framework/python/src/net_orc/listener.py @@ -1,9 +1,23 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Intercepts network traffic between network services and the device under test.""" import threading from scapy.all import AsyncSniffer, DHCP, get_if_hwaddr -import logger -from network_event import NetworkEvent +from net_orc.network_event import NetworkEvent +from common import logger LOGGER = logger.get_logger('listener') diff --git a/framework/python/src/net_orc/network_device.py b/framework/python/src/net_orc/network_device.py new file mode 100644 index 000000000..f17ac0f0d --- /dev/null +++ b/framework/python/src/net_orc/network_device.py @@ -0,0 +1,24 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Track device object information.""" +from dataclasses import dataclass + + +@dataclass +class NetworkDevice: + """Represents a physical device and it's configuration.""" + + mac_addr: str + ip_addr: str = None diff --git a/framework/python/src/net_orc/network_event.py b/framework/python/src/net_orc/network_event.py new file mode 100644 index 000000000..204c97a0a --- /dev/null +++ b/framework/python/src/net_orc/network_event.py @@ -0,0 +1,23 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Specify the various types of network events to be reported.""" +from enum import Enum + + +class NetworkEvent(Enum): + """All possible network events.""" + DEVICE_DISCOVERED = 1 + DEVICE_STABLE = 2 + DHCP_LEASE_ACK = 3 diff --git a/net_orc/python/src/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py similarity index 74% rename from net_orc/python/src/network_orchestrator.py rename to framework/python/src/net_orc/network_orchestrator.py index 53a94b795..643dc4def 100644 --- a/net_orc/python/src/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -1,3 +1,16 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Network orchestrator is responsible for managing all of the virtual network services""" import getpass @@ -12,21 +25,24 @@ import threading import docker from docker.types import Mount -import logger -import util -from listener import Listener -from network_device import NetworkDevice -from network_event import NetworkEvent -from network_validator import NetworkValidator +from common import logger +from common import util +from net_orc.listener import Listener +from net_orc.network_device import NetworkDevice +from net_orc.network_event import NetworkEvent +from net_orc.network_validator import NetworkValidator +from net_orc.ovs_control import OVSControl +from net_orc.ip_control import IPControl LOGGER = logger.get_logger('net_orc') -CONFIG_FILE = 'conf/system.json' -EXAMPLE_CONFIG_FILE = 'conf/system.json.example' +CONFIG_FILE = 'local/system.json' +EXAMPLE_CONFIG_FILE = 'local/system.json.example' RUNTIME_DIR = 'runtime' -DEVICES_DIR = 'devices' +TEST_DIR = 'test' MONITOR_PCAP = 'monitor.pcap' NET_DIR = 'runtime/network' -NETWORK_MODULES_DIR = 'network/modules' +#NETWORK_MODULES_DIR = 'network/modules' +NETWORK_MODULES_DIR = 'modules/network' NETWORK_MODULE_METADATA = 'conf/module_config.json' DEVICE_BRIDGE = 'tr-d' INTERNET_BRIDGE = 'tr-c' @@ -67,17 +83,24 @@ def __init__(self, self.async_monitor = async_monitor self._path = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) self.validator = NetworkValidator() shutil.rmtree(os.path.join(os.getcwd(), NET_DIR), ignore_errors=True) self.network_config = NetworkConfig() self.load_config(config_file) + self._ovs = OVSControl() + self._ip_ctrl = IPControl() def start(self): """Start the network orchestrator.""" - LOGGER.info('Starting Network Orchestrator') + LOGGER.debug('Starting network orchestrator') + + self._host_user = self._get_host_user() + # Get all components ready self.load_network_modules() @@ -109,6 +132,9 @@ def start_network(self): # Get network ready (via Network orchestrator) LOGGER.info('Network is ready.') + def start_listener(self): + self.listener.start_listener() + def stop(self, kill=False): """Stop the network orchestrator.""" self.stop_validator(kill=kill) @@ -142,7 +168,7 @@ def load_config(self, config_file=None): if not os.path.isfile(self._config_file): LOGGER.error('Configuration file is not present at ' + config_file) - LOGGER.info('An example is present in '+ EXAMPLE_CONFIG_FILE) + LOGGER.info('An example is present in ' + EXAMPLE_CONFIG_FILE) sys.exit(1) LOGGER.info('Loading config file: ' + os.path.abspath(self._config_file)) @@ -153,31 +179,37 @@ def load_config(self, config_file=None): def _device_discovered(self, mac_addr): LOGGER.debug( - f'Discovered device {mac_addr}. Waiting for device to obtain IP') + f'Discovered device {mac_addr}. Waiting for device to obtain IP') device = self._get_device(mac_addr=mac_addr) - os.makedirs( - os.path.join(RUNTIME_DIR, DEVICES_DIR, device.mac_addr.replace(':', - ''))) - timeout = time.time() + self._startup_timeout + device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, + device.mac_addr.replace(':', '')) + os.makedirs(device_runtime_dir) + util.run_command(f'chown -R {self._host_user} {device_runtime_dir}') - while time.time() < timeout: - if device.ip_addr is None: - time.sleep(3) - else: - break + packet_capture = sniff(iface=self._dev_intf, + timeout=self._startup_timeout, + stop_filter=self._device_has_ip) + wrpcap( + os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), + 'startup.pcap'), packet_capture) if device.ip_addr is None: LOGGER.info( - f'Timed out whilst waiting for {mac_addr} to obtain an IP address') + f'Timed out whilst waiting for {mac_addr} to obtain an IP address') return - LOGGER.info( - f'Device with mac addr {device.mac_addr} has obtained IP address ' - f'{device.ip_addr}') + f'Device with mac addr {device.mac_addr} has obtained IP address ' + f'{device.ip_addr}') self._start_device_monitor(device) + def _device_has_ip(self, packet): + device = self._get_device(mac_addr=packet.src) + if device is None or device.ip_addr is None: + return False + return True + def _dhcp_lease_ack(self, packet): mac_addr = packet[BOOTP].chaddr.hex(':')[0:17] device = self._get_device(mac_addr=mac_addr) @@ -186,13 +218,12 @@ def _dhcp_lease_ack(self, packet): def _start_device_monitor(self, device): """Start a timer until the steady state has been reached and callback the steady state method for this device.""" - LOGGER.info( - f'Monitoring device with mac addr {device.mac_addr} ' - f'for {str(self._monitor_period)} seconds') + LOGGER.info(f'Monitoring device with mac addr {device.mac_addr} ' + f'for {str(self._monitor_period)} seconds') packet_capture = sniff(iface=self._dev_intf, timeout=self._monitor_period) wrpcap( - os.path.join(RUNTIME_DIR, DEVICES_DIR, device.mac_addr.replace(':', ''), + os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), 'monitor.pcap'), packet_capture) self.listener.call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) @@ -293,9 +324,8 @@ def _ci_post_network_create(self): util.run_command(f'ip link set dev {INTERNET_BRIDGE} up') util.run_command(f'dhclient {INTERNET_BRIDGE}') util.run_command('ip route del default via 10.1.0.1') - util.run_command( - f'ip route add default via {self._gateway} ' - f'src {self._ipv4[:-3]} metric 100 dev {INTERNET_BRIDGE}') + util.run_command(f'ip route add default via {self._gateway} ' + f'src {self._ipv4[:-3]} metric 100 dev {INTERNET_BRIDGE}') def create_net(self): LOGGER.info('Creating baseline network') @@ -309,28 +339,14 @@ def create_net(self): if self._single_intf: self._ci_pre_network_create() - # Create data plane - util.run_command('ovs-vsctl add-br ' + DEVICE_BRIDGE) - - # Create control plane - util.run_command('ovs-vsctl add-br ' + INTERNET_BRIDGE) - - # Add external interfaces to data and control plane - util.run_command('ovs-vsctl add-port ' + DEVICE_BRIDGE + ' ' + - self._dev_intf) - util.run_command('ovs-vsctl add-port ' + INTERNET_BRIDGE + ' ' + - self._int_intf) - - # Enable forwarding of eapol packets - util.run_command('ovs-ofctl add-flow ' + DEVICE_BRIDGE + - ' \'table=0, dl_dst=01:80:c2:00:00:03, actions=flood\'') - # Remove IP from internet adapter util.run_command('ifconfig ' + self._int_intf + ' 0.0.0.0') - # Set ports up - util.run_command('ip link set dev ' + DEVICE_BRIDGE + ' up') - util.run_command('ip link set dev ' + INTERNET_BRIDGE + ' up') + # Setup the virtual network + if not self._ovs.create_baseline_net(verify=True): + LOGGER.error('Baseline network validation failed.') + self.stop() + sys.exit(1) if self._single_intf: self._ci_post_network_create() @@ -342,7 +358,6 @@ def create_net(self): [NetworkEvent.DEVICE_DISCOVERED]) self.listener.register_callback(self._dhcp_lease_ack, [NetworkEvent.DHCP_LEASE_ACK]) - self.listener.start_listener() def load_network_modules(self): """Load network modules from module_config.json.""" @@ -367,8 +382,9 @@ def _load_network_module(self, module_dir): # Load module information with open(os.path.join(self._path, net_modules_dir, module_dir, - NETWORK_MODULE_METADATA), 'r', - encoding='UTF-8') as module_file_open: + NETWORK_MODULE_METADATA), + 'r', + encoding='UTF-8') as module_file_open: net_module_json = json.load(module_file_open) net_module.name = net_module_json['config']['meta']['name'] @@ -400,6 +416,10 @@ def _load_network_module(self, module_dir): net_module.enable_container = net_module_json['config']['docker'][ 'enable_container'] + # Determine if this is a template + if 'template' in net_module_json['config']['docker']: + net_module.template = net_module_json['config']['docker']['template'] + # Load network service networking configuration if net_module.enable_container: @@ -419,13 +439,14 @@ def _load_network_module(self, module_dir): net_module.net_config.ip_index] net_module.net_config.ipv6_network = self.network_config.ipv6_network - self._net_modules.append(net_module) + self._net_modules.append(net_module) return net_module def build_network_modules(self): LOGGER.info('Building network modules...') for net_module in self._net_modules: - self._build_module(net_module) + if not net_module.template: + self._build_module(net_module) def _build_module(self, net_module): LOGGER.debug('Building network module ' + net_module.dir_name) @@ -443,13 +464,6 @@ def _get_network_module(self, name): return net_module return None - # Start the OVS network module - # This should always be called before loading all - # other modules to allow for a properly setup base - # network - def _start_ovs_module(self): - self._start_network_service(self._get_network_module('OVS')) - def _start_network_service(self, net_module): LOGGER.debug('Starting net service ' + net_module.display_name) @@ -468,7 +482,7 @@ def _start_network_service(self, net_module): privileged=True, detach=True, mounts=net_module.mounts, - environment={'HOST_USER': getpass.getuser()}) + environment={'HOST_USER': self._host_user}) except docker.errors.ContainerError as error: LOGGER.error('Container run error') LOGGER.error(error) @@ -476,6 +490,46 @@ def _start_network_service(self, net_module): if network != 'host': self._attach_service_to_network(net_module) + def _get_host_user(self): + user = self._get_os_user() + + # If primary method failed, try secondary + if user is None: + user = self._get_user() + + LOGGER.debug("Network orchestrator host user: " + user) + return user + + def _get_os_user(self): + user = None + try: + user = os.getlogin() + except OSError as e: + # Handle the OSError exception + LOGGER.error("An OS error occurred while retrieving the login name.") + except Exception as e: + # Catch any other unexpected exceptions + LOGGER.error("An exception occurred:", e) + return user + + def _get_user(self): + user = None + try: + user = getpass.getuser() + except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: + # Handle specific exceptions individually + if isinstance(e, KeyError): + LOGGER.error("USER environment variable not set or unavailable.") + elif isinstance(e, ImportError): + LOGGER.error("Unable to import the getpass module.") + elif isinstance(e, ModuleNotFoundError): + LOGGER.error("The getpass module was not found.") + elif isinstance(e, OSError): + LOGGER.error("An OS error occurred while retrieving the username.") + else: + LOGGER.error("An exception occurred:", e) + return user + def _stop_service_module(self, net_module, kill=False): LOGGER.debug('Stopping Service container ' + net_module.container_name) try: @@ -521,16 +575,12 @@ def start_network_services(self): for net_module in self._net_modules: - # TODO: There should be a better way of doing this - # Do not try starting OVS module again, as it should already be running - if 'OVS' != net_module.display_name: - - # Network modules may just be Docker images, - # so we do not want to start them as containers - if not net_module.enable_container: - continue + # Network modules may just be Docker images, + # so we do not want to start them as containers + if not net_module.enable_container: + continue - self._start_network_service(net_module) + self._start_network_service(net_module) LOGGER.info('All network services are running') self._check_network_services() @@ -555,7 +605,7 @@ def attach_test_module_to_network(self, test_module): container_intf) # Add bridge interface to device bridge - util.run_command('ovs-vsctl add-port ' + DEVICE_BRIDGE + ' ' + bridge_intf) + self._ovs.add_port(port=bridge_intf, bridge_name=DEVICE_BRIDGE) # Get PID for running container # TODO: Some error checking around missing PIDs might be required @@ -615,48 +665,25 @@ def _attach_service_to_network(self, net_module): # Container network namespace name container_net_ns = 'tr-ctns-' + net_module.dir_name - # Create interface pair - util.run_command('ip link add ' + bridge_intf + ' type veth peer name ' + - container_intf) + # Resolve the interface information + mac_addr = '9a:02:57:1e:8f:' + str(net_module.net_config.ip_index) + ipv4_addr = net_module.net_config.get_ipv4_addr_with_prefix() + ipv6_addr = net_module.net_config.get_ipv6_addr_with_prefix() + + # Add and configure the interface container + if not self._ip_ctrl.configure_container_interface( + bridge_intf, container_intf, "veth0", container_net_ns, mac_addr, + net_module.container_name, ipv4_addr, ipv6_addr): + LOGGER.error('Failed to configure local networking for ' + + net_module.name + '. Exiting.') + sys.exit(1) # Add bridge interface to device bridge - util.run_command('ovs-vsctl add-port ' + DEVICE_BRIDGE + ' ' + bridge_intf) - - # Get PID for running container - # TODO: Some error checking around missing PIDs might be required - container_pid = util.run_command('docker inspect -f {{.State.Pid}} ' + - net_module.container_name)[0] - - # Create symlink for container network namespace - util.run_command('ln -sf /proc/' + container_pid + - '/ns/net /var/run/netns/' + container_net_ns) - - # Attach container interface to container network namespace - util.run_command('ip link set ' + container_intf + ' netns ' + - container_net_ns) - - # Rename container interface name to veth0 - util.run_command('ip netns exec ' + container_net_ns + ' ip link set dev ' + - container_intf + ' name veth0') - - # Set MAC address of container interface - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev veth0 address 9a:02:57:1e:8f:' + - str(net_module.net_config.ip_index)) - - # Set IP address of container interface - util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + - net_module.net_config.get_ipv4_addr_with_prefix() + - ' dev veth0') - - util.run_command('ip netns exec ' + container_net_ns + ' ip addr add ' + - net_module.net_config.get_ipv6_addr_with_prefix() + - ' dev veth0') - - # Set interfaces up - util.run_command('ip link set dev ' + bridge_intf + ' up') - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev veth0 up') + if self._ovs.add_port(port=bridge_intf, bridge_name=DEVICE_BRIDGE): + if not self._ovs.port_exists(bridge_name=DEVICE_BRIDGE, port=bridge_intf): + LOGGER.error('Failed to add ' + net_module.name + ' to device bridge ' + + DEVICE_BRIDGE + '. Exiting.') + sys.exit(1) if net_module.net_config.enable_wan: LOGGER.debug('Attaching net service ' + net_module.display_name + @@ -670,31 +697,19 @@ def _attach_service_to_network(self, net_module): # tr-cti-dhcp (Test Run Container Interface for DHCP container) container_intf = 'tr-cti-' + net_module.dir_name - # Create interface pair - util.run_command('ip link add ' + bridge_intf + ' type veth peer name ' + - container_intf) + if not self._ip_ctrl.configure_container_interface( + bridge_intf, container_intf, "eth1", container_net_ns, mac_addr): + LOGGER.error('Failed to configure internet networking for ' + + net_module.name + '. Exiting.') + sys.exit(1) # Attach bridge interface to internet bridge - util.run_command('ovs-vsctl add-port ' + INTERNET_BRIDGE + ' ' + - bridge_intf) - - # Attach container interface to container network namespace - util.run_command('ip link set ' + container_intf + ' netns ' + - container_net_ns) - - # Rename container interface name to eth1 - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev ' + container_intf + ' name eth1') - - # Set MAC address of container interface - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev eth1 address 9a:02:57:1e:8f:0' + - str(net_module.net_config.ip_index)) - - # Set interfaces up - util.run_command('ip link set dev ' + bridge_intf + ' up') - util.run_command('ip netns exec ' + container_net_ns + - ' ip link set dev eth1 up') + if self._ovs.add_port(port=bridge_intf, bridge_name=INTERNET_BRIDGE): + if not self._ovs.port_exists(bridge_name=INTERNET_BRIDGE, + port=bridge_intf): + LOGGER.error('Failed to add ' + net_module.name + + ' to internet bridge ' + DEVICE_BRIDGE + '. Exiting.') + sys.exit(1) def restore_net(self): @@ -714,11 +729,11 @@ def restore_net(self): except Exception: # pylint: disable=W0703 continue - # Delete data plane - util.run_command('ovs-vsctl --if-exists del-br tr-d') + # Clear the virtual network + self._ovs.restore_net() - # Delete control plane - util.run_command('ovs-vsctl --if-exists del-br tr-c') + # Clean up any existing network artifacts + self._ip_ctrl.clean_all() # Restart internet interface if util.interface_exists(self._int_intf): @@ -739,6 +754,7 @@ def __init__(self): self.container = None self.container_name = None self.image_name = None + self.template = False # Absolute path self.dir = None diff --git a/net_orc/python/src/network_validator.py b/framework/python/src/net_orc/network_validator.py similarity index 74% rename from net_orc/python/src/network_validator.py rename to framework/python/src/net_orc/network_validator.py index 83ca6f671..a4c51eb2d 100644 --- a/net_orc/python/src/network_validator.py +++ b/framework/python/src/net_orc/network_validator.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Holds logic for validation of network services prior to runtime.""" import json import os @@ -6,15 +20,15 @@ import docker from docker.types import Mount import getpass -import logger -import util +from common import logger +from common import util LOGGER = logger.get_logger('validator') OUTPUT_DIR = 'runtime/validation' -DEVICES_DIR = 'network/devices' +DEVICES_DIR = 'modules/devices' DEVICE_METADATA = 'conf/module_config.json' DEVICE_BRIDGE = 'tr-d' -CONF_DIR = 'conf' +CONF_DIR = 'local' CONF_FILE = 'system.json' @@ -24,8 +38,9 @@ class NetworkValidator: def __init__(self): self._net_devices = [] - self._path = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + self._path = os.path.dirname(os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) self._device_dir = os.path.join(self._path, DEVICES_DIR) @@ -33,16 +48,22 @@ def __init__(self): def start(self): """Start the network validator.""" - LOGGER.info('Starting validator') + LOGGER.debug('Starting validator') + + # Setup the output directory + host_user = self._get_host_user() + os.makedirs(OUTPUT_DIR, exist_ok=True) + util.run_command(f'chown -R {host_user} {OUTPUT_DIR}') + self._load_devices() self._build_network_devices() self._start_network_devices() def stop(self, kill=False): """Stop the network validator.""" - LOGGER.info('Stopping validator') + LOGGER.debug('Stopping validator') self._stop_network_devices(kill) - LOGGER.info('Validator stopped') + LOGGER.debug('Validator stopped') def _build_network_devices(self): LOGGER.debug('Building network validators...') @@ -65,7 +86,7 @@ def _build_device(self, net_device): def _load_devices(self): - LOGGER.info(f'Loading validators from {DEVICES_DIR}') + LOGGER.info(f'Loading validators from {self._device_dir}') loaded_devices = 'Loaded the following validators: ' @@ -136,7 +157,7 @@ def _start_network_device(self, device): privileged=True, detach=True, mounts=device.mounts, - environment={'HOST_USER': getpass.getuser()}) + environment={'HOST_USER': self._get_host_user()}) except docker.errors.ContainerError as error: LOGGER.error('Container run error') LOGGER.error(error) @@ -153,6 +174,46 @@ def _start_network_device(self, device): LOGGER.info('Validation device ' + device.name + ' has finished') + def _get_host_user(self): + user = self._get_os_user() + + # If primary method failed, try secondary + if user is None: + user = self._get_user() + + LOGGER.debug(f'Network validator host user: {user}') + return user + + def _get_os_user(self): + user = None + try: + user = os.getlogin() + except OSError: + # Handle the OSError exception + LOGGER.error('An OS error occurred while retrieving the login name.') + except Exception as error: + # Catch any other unexpected exceptions + LOGGER.error('An exception occurred:', error) + return user + + def _get_user(self): + user = None + try: + user = getpass.getuser() + except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: + # Handle specific exceptions individually + if isinstance(e, KeyError): + LOGGER.error("USER environment variable not set or unavailable.") + elif isinstance(e, ImportError): + LOGGER.error("Unable to import the getpass module.") + elif isinstance(e, ModuleNotFoundError): + LOGGER.error("The getpass module was not found.") + elif isinstance(e, OSError): + LOGGER.error("An OS error occurred while retrieving the username.") + else: + LOGGER.error("An exception occurred:", e) + return user + def _get_device_status(self, module): container = self._get_device_container(module) if container is not None: diff --git a/framework/python/src/net_orc/ovs_control.py b/framework/python/src/net_orc/ovs_control.py new file mode 100644 index 000000000..83823e8fa --- /dev/null +++ b/framework/python/src/net_orc/ovs_control.py @@ -0,0 +1,186 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OVS Control Module""" +import json +import os +from common import logger +from common import util + +CONFIG_FILE = 'local/system.json' +DEVICE_BRIDGE = 'tr-d' +INTERNET_BRIDGE = 'tr-c' +LOGGER = logger.get_logger('ovs_ctrl') + + +class OVSControl: + """OVS Control""" + + def __init__(self): + self._int_intf = None + self._dev_intf = None + self._load_config() + + def add_bridge(self, bridge_name): + LOGGER.debug('Adding OVS bridge: ' + bridge_name) + # Create the bridge using ovs-vsctl commands + # Uses the --may-exist option to prevent failures + # if this bridge already exists by this name it won't fail + # and will not modify the existing bridge + success = util.run_command('ovs-vsctl --may-exist add-br ' + bridge_name) + return success + + def add_flow(self, bridge_name, flow): + # Add a flow to the bridge using ovs-ofctl commands + LOGGER.debug(f'Adding flow {flow} to bridge: {bridge_name}') + success = util.run_command(f'ovs-ofctl add-flow {bridge_name} \'{flow}\'') + return success + + def add_port(self, port, bridge_name): + LOGGER.debug('Adding port ' + port + ' to OVS bridge: ' + bridge_name) + # Add a port to the bridge using ovs-vsctl commands + # Uses the --may-exist option to prevent failures + # if this port already exists on the bridge and will not + # modify the existing bridge + success = util.run_command(f"""ovs-vsctl --may-exist + add-port {bridge_name} {port}""") + return success + + def get_bridge_ports(self, bridge_name): + # Get a list of all the ports on a bridge + response = util.run_command(f'ovs-vsctl list-ports {bridge_name}', + output=True) + return response[0].splitlines() + + def bridge_exists(self, bridge_name): + # Check if a bridge exists by the name provided + LOGGER.debug(f'Checking if {bridge_name} exists') + success = util.run_command(f'ovs-vsctl br-exists {bridge_name}') + return success + + def port_exists(self, bridge_name, port): + # Check if a port exists on a specified bridge + LOGGER.debug(f'Checking if {bridge_name} exists') + resp = util.run_command(f'ovs-vsctl port-to-br {port}', True) + return resp[0] == bridge_name + + def validate_baseline_network(self): + # Verify the OVS setup of the virtual network + LOGGER.debug('Validating baseline network') + + # Verify the device bridge + dev_bridge = self.verify_bridge(DEVICE_BRIDGE, [self._dev_intf]) + LOGGER.debug('Device bridge verified: ' + str(dev_bridge)) + + # Verify the internet bridge + int_bridge = self.verify_bridge(INTERNET_BRIDGE, [self._int_intf]) + LOGGER.debug('Internet bridge verified: ' + str(int_bridge)) + + return dev_bridge and int_bridge + + def verify_bridge(self, bridge_name, ports): + LOGGER.debug('Verifying bridge: ' + bridge_name) + verified = True + if self.bridge_exists(bridge_name): + bridge_ports = self.get_bridge_ports(bridge_name) + LOGGER.debug('Checking bridge for ports: ' + str(ports)) + for port in ports: + if port not in bridge_ports: + verified = False + break + else: + verified = False + return verified + + def create_baseline_net(self, verify=True): + LOGGER.debug('Creating baseline network') + + # Remove IP from internet adapter + self.set_interface_ip(interface=self._int_intf, ip_addr='0.0.0.0') + + # Create data plane + self.add_bridge(DEVICE_BRIDGE) + + # Create control plane + self.add_bridge(INTERNET_BRIDGE) + + # Remove IP from internet adapter + self.set_interface_ip(self._int_intf, '0.0.0.0') + + # Add external interfaces to data and control plane + self.add_port(self._dev_intf, DEVICE_BRIDGE) + self.add_port(self._int_intf, INTERNET_BRIDGE) + + # Enable forwarding of eapol packets + self.add_flow(bridge_name=DEVICE_BRIDGE, + flow='table=0, dl_dst=01:80:c2:00:00:03, actions=flood') + + # Set ports up + self.set_bridge_up(DEVICE_BRIDGE) + self.set_bridge_up(INTERNET_BRIDGE) + + self.show_config() + + if verify: + return self.validate_baseline_network() + else: + return None + + def delete_bridge(self, bridge_name): + LOGGER.debug('Deleting OVS Bridge: ' + bridge_name) + # Delete the bridge using ovs-vsctl commands + # Uses the --if-exists option to prevent failures + # if this bridge does not exists + success = util.run_command('ovs-vsctl --if-exists del-br ' + bridge_name) + return success + + def _load_config(self): + path = os.path.dirname(os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) + config_file = os.path.join(path, CONFIG_FILE) + LOGGER.debug('Loading configuration: ' + config_file) + with open(config_file, 'r', encoding='utf-8') as conf_file: + config_json = json.load(conf_file) + self._int_intf = config_json['network']['internet_intf'] + self._dev_intf = config_json['network']['device_intf'] + LOGGER.debug('Configuration loaded') + LOGGER.debug('Internet interface: ' + self._int_intf) + LOGGER.debug('Device interface: ' + self._dev_intf) + + def restore_net(self): + LOGGER.debug('Restoring network...') + # Delete data plane + self.delete_bridge(DEVICE_BRIDGE) + + # Delete control plane + self.delete_bridge(INTERNET_BRIDGE) + + LOGGER.debug('Network is restored') + + def show_config(self): + LOGGER.debug('Show current config of OVS') + success = util.run_command('ovs-vsctl show', output=True) + LOGGER.debug(f'OVS Config\n{success[0]}') + return success + + def set_bridge_up(self, bridge_name): + LOGGER.debug('Setting bridge device to up state: ' + bridge_name) + success = util.run_command('ip link set dev ' + bridge_name + ' up') + return success + + def set_interface_ip(self, interface, ip_addr): + LOGGER.debug('Setting interface ' + interface + ' to ' + ip_addr) + # Remove IP from internet adapter + util.run_command(f'ifconfig {interface} {ip_addr}') diff --git a/test_orc/python/src/module.py b/framework/python/src/test_orc/module.py similarity index 50% rename from test_orc/python/src/module.py rename to framework/python/src/test_orc/module.py index 72791f86e..185940dd8 100644 --- a/test_orc/python/src/module.py +++ b/framework/python/src/test_orc/module.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Represemts a test module.""" from dataclasses import dataclass from docker.models.containers import Container diff --git a/test_orc/python/src/runner.py b/framework/python/src/test_orc/runner.py similarity index 65% rename from test_orc/python/src/runner.py rename to framework/python/src/test_orc/runner.py index d82935057..ed3b9059a 100644 --- a/test_orc/python/src/runner.py +++ b/framework/python/src/test_orc/runner.py @@ -1,41 +1,55 @@ -"""Provides high level management of the test orchestrator.""" -import time -import logger - -LOGGER = logger.get_logger('runner') - - -class Runner: - """Holds the state of the testing for one device.""" - - def __init__(self, test_orc, device): - self._test_orc = test_orc - self._device = device - - def run(self): - self._run_test_modules() - - def _run_test_modules(self): - """Iterates through each test module and starts the container.""" - LOGGER.info('Running test modules...') - for module in self._test_modules: - self.run_test_module(module) - LOGGER.info('All tests complete') - - def run_test_module(self, module): - """Start the test container and extract the results.""" - - if module is None or not module.enable_container: - return - - self._test_orc.start_test_module(module) - - # Determine the module timeout time - test_module_timeout = time.time() + module.timeout - status = self._test_orc.get_module_status(module) - - while time.time() < test_module_timeout and status == 'running': - time.sleep(1) - status = self._test_orc.get_module_status(module) - - LOGGER.info(f'Test module {module.display_name} has finished') +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides high level management of the test orchestrator.""" +import time +import logger + +LOGGER = logger.get_logger('runner') + + +class Runner: + """Holds the state of the testing for one device.""" + + def __init__(self, test_orc, device): + self._test_orc = test_orc + self._device = device + + def run(self): + self._run_test_modules() + + def _run_test_modules(self): + """Iterates through each test module and starts the container.""" + LOGGER.info('Running test modules...') + for module in self._test_modules: + self.run_test_module(module) + LOGGER.info('All tests complete') + + def run_test_module(self, module): + """Start the test container and extract the results.""" + + if module is None or not module.enable_container: + return + + self._test_orc.start_test_module(module) + + # Determine the module timeout time + test_module_timeout = time.time() + module.timeout + status = self._test_orc.get_module_status(module) + + while time.time() < test_module_timeout and status == 'running': + time.sleep(1) + status = self._test_orc.get_module_status(module) + + LOGGER.info(f'Test module {module.display_name} has finished') diff --git a/test_orc/python/src/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py similarity index 68% rename from test_orc/python/src/test_orchestrator.py rename to framework/python/src/test_orc/test_orchestrator.py index 4b65bae12..58c1944f8 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Provides high level management of the test orchestrator.""" import getpass import os @@ -6,13 +20,14 @@ import shutil import docker from docker.types import Mount -import logger -from module import TestModule +from common import logger +from test_orc.module import TestModule +from common import util LOG_NAME = "test_orc" LOGGER = logger.get_logger("test_orc") -RUNTIME_DIR = "runtime" -TEST_MODULES_DIR = "modules" +RUNTIME_DIR = "runtime/test" +TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" @@ -23,19 +38,31 @@ def __init__(self, net_orc): self._test_modules = [] self._module_config = None self._net_orc = net_orc + self._test_in_progress = False - self._path = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + self._path = os.path.dirname(os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) # Resolve the path to the test-run folder - self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) + #self._root_path = os.path.abspath(os.path.join(self._path, os.pardir)) + + + self._root_path = os.path.dirname(os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))) shutil.rmtree(os.path.join(self._root_path, RUNTIME_DIR), ignore_errors=True) - os.makedirs(os.path.join(self._root_path, RUNTIME_DIR), exist_ok=True) def start(self): - LOGGER.info("Starting Test Orchestrator") + LOGGER.debug("Starting test orchestrator") + + # Setup the output directory + self._host_user = self._get_host_user() + os.makedirs(RUNTIME_DIR, exist_ok=True) + util.run_command(f'chown -R {self._host_user} {RUNTIME_DIR}') + self._load_test_modules() self.build_test_modules() @@ -45,14 +72,18 @@ def stop(self): def run_test_modules(self, device): """Iterates through each test module and starts the container.""" + self._test_in_progress = True LOGGER.info( f"Running test modules on device with mac addr {device.mac_addr}") for module in self._test_modules: self._run_test_module(module, device) LOGGER.info("All tests complete") - LOGGER.info(f"""Completed running test modules on device - with mac addr {device.mac_addr}""") + LOGGER.info( + f"""Completed running test \ +modules on device with mac \ +addr {device.mac_addr}""") self._generate_results(device) + self._test_in_progress = False def _generate_results(self, device): results = {} @@ -74,7 +105,7 @@ def _generate_results(self, device): results[module.name] = module_results except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: - LOGGER.error("Module Results Errror " + module.name) + LOGGER.error("Error occured whilst running module " + module.name) LOGGER.debug(results_error) out_file = os.path.join( @@ -82,8 +113,12 @@ def _generate_results(self, device): "runtime/test/" + device.mac_addr.replace(":", "") + "/results.json") with open(out_file, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) + util.run_command(f'chown -R {self._host_user} {out_file}') return results + def test_in_progress(self): + return self._test_in_progress + def _is_module_enabled(self, module, device): enabled = True if device.test_modules is not None: @@ -111,6 +146,16 @@ def _run_test_module(self, module, device): network_runtime_dir = os.path.join(self._root_path, "runtime/network") os.makedirs(container_runtime_dir) + device_startup_capture = os.path.join( + self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + + "/startup.pcap") + util.run_command(f'chown -R {self._host_user} {device_startup_capture}') + + device_monitor_capture = os.path.join( + self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + + "/monitor.pcap") + util.run_command(f'chown -R {self._host_user} {device_monitor_capture}') + client = docker.from_env() module.container = client.containers.run( @@ -129,9 +174,17 @@ def _run_test_module(self, module, device): source=network_runtime_dir, type="bind", read_only=True), + Mount(target="/runtime/device/startup.pcap", + source=device_startup_capture, + type="bind", + read_only=True), + Mount(target="/runtime/device/monitor.pcap", + source=device_monitor_capture, + type="bind", + read_only=True), ], environment={ - "HOST_USER": getpass.getuser(), + "HOST_USER": self._host_user, "DEVICE_MAC": device.mac_addr, "DEVICE_TEST_MODULES": device.test_modules, "IPV4_SUBNET": self._net_orc.network_config.ipv4_network, @@ -184,6 +237,47 @@ def _get_module_container(self, module): LOGGER.error(error) return container + def _get_host_user(self): + user = self._get_os_user() + + # If primary method failed, try secondary + if user is None: + user = self._get_user() + + LOGGER.debug("Test orchestrator host user: " + user) + return user + + def _get_os_user(self): + user = None + try: + user = os.getlogin() + except OSError as e: + # Handle the OSError exception + LOGGER.error("An OS error occurred while retrieving the login name.") + except Exception as e: + # Catch any other unexpected exceptions + LOGGER.error("An exception occurred:", e) + return user + + def _get_user(self): + user = None + try: + user = getpass.getuser() + except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: + # Handle specific exceptions individually + if isinstance(e, KeyError): + LOGGER.error("USER environment variable not set or unavailable.") + elif isinstance(e, ImportError): + LOGGER.error("Unable to import the getpass module.") + elif isinstance(e, ModuleNotFoundError): + LOGGER.error("The getpass module was not found.") + elif isinstance(e, OSError): + LOGGER.error("An OS error occurred while retrieving the username.") + else: + LOGGER.error("An exception occurred:", e) + return user + + def _load_test_modules(self): """Load network modules from module_config.json.""" LOGGER.debug("Loading test modules from /" + TEST_MODULES_DIR) diff --git a/framework/requirements.txt b/framework/requirements.txt index ca56948f4..03eab9796 100644 --- a/framework/requirements.txt +++ b/framework/requirements.txt @@ -1 +1,8 @@ -requests<2.29.0 \ No newline at end of file +# Requirements for the core module +requests<2.29.0 + +# Requirements for the net_orc module +docker +ipaddress +netifaces +scapy \ No newline at end of file diff --git a/local/.gitignore b/local/.gitignore new file mode 100644 index 000000000..4fb365c03 --- /dev/null +++ b/local/.gitignore @@ -0,0 +1,2 @@ +system.json +devices \ No newline at end of file diff --git a/conf/system.json.example b/local/system.json.example similarity index 100% rename from conf/system.json.example rename to local/system.json.example diff --git a/net_orc/network/devices/faux-dev/bin/get_default_gateway b/modules/devices/faux-dev/bin/get_default_gateway similarity index 100% rename from net_orc/network/devices/faux-dev/bin/get_default_gateway rename to modules/devices/faux-dev/bin/get_default_gateway diff --git a/net_orc/network/devices/faux-dev/bin/start_dhcp_client b/modules/devices/faux-dev/bin/start_dhcp_client similarity index 100% rename from net_orc/network/devices/faux-dev/bin/start_dhcp_client rename to modules/devices/faux-dev/bin/start_dhcp_client diff --git a/net_orc/network/devices/faux-dev/bin/start_network_service b/modules/devices/faux-dev/bin/start_network_service similarity index 79% rename from net_orc/network/devices/faux-dev/bin/start_network_service rename to modules/devices/faux-dev/bin/start_network_service index b727d2091..80a587684 100644 --- a/net_orc/network/devices/faux-dev/bin/start_network_service +++ b/modules/devices/faux-dev/bin/start_network_service @@ -22,12 +22,12 @@ else fi #Create and set permissions on the output files -LOG_FILE=/runtime/validation/$MODULE_NAME.log -RESULT_FILE=/runtime/validation/result.json +OUTPUT_DIR=/runtime/validation/ +LOG_FILE=$OUTPUT_DIR/$MODULE_NAME.log +RESULT_FILE=$OUTPUT_DIR/result.json touch $LOG_FILE touch $RESULT_FILE -chown $HOST_USER:$HOST_USER $LOG_FILE -chown $HOST_USER:$HOST_USER $RESULT_FILE +chown -R $HOST_USER $OUTPUT_DIR # Start dhclient $BIN_DIR/start_dhcp_client $INTF diff --git a/net_orc/network/devices/faux-dev/conf/module_config.json b/modules/devices/faux-dev/conf/module_config.json similarity index 100% rename from net_orc/network/devices/faux-dev/conf/module_config.json rename to modules/devices/faux-dev/conf/module_config.json diff --git a/net_orc/network/devices/faux-dev/faux-dev.Dockerfile b/modules/devices/faux-dev/faux-dev.Dockerfile similarity index 65% rename from net_orc/network/devices/faux-dev/faux-dev.Dockerfile rename to modules/devices/faux-dev/faux-dev.Dockerfile index 1686341b5..0a4f02f38 100644 --- a/net_orc/network/devices/faux-dev/faux-dev.Dockerfile +++ b/modules/devices/faux-dev/faux-dev.Dockerfile @@ -1,6 +1,9 @@ # Image name: test-run/faux-dev FROM test-run/base:latest +ARG MODULE_NAME=faux-dev +ARG MODULE_DIR=modules/devices/$MODULE_NAME + #Update and get all additional requirements not contained in the base image RUN apt-get update --fix-missing @@ -11,10 +14,10 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get install -y isc-dhcp-client ntp ntpdate # Copy over all configuration files -COPY network/devices/faux-dev/conf /testrun/conf +COPY $MODULE_DIR/conf /testrun/conf -# Load device binary files -COPY network/devices/faux-dev/bin /testrun/bin +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY network/devices/faux-dev/python /testrun/python \ No newline at end of file +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/net_orc/network/devices/faux-dev/python/src/dhcp_check.py b/modules/devices/faux-dev/python/src/dhcp_check.py similarity index 82% rename from net_orc/network/devices/faux-dev/python/src/dhcp_check.py rename to modules/devices/faux-dev/python/src/dhcp_check.py index 82dd6e31f..565e33308 100644 --- a/net_orc/network/devices/faux-dev/python/src/dhcp_check.py +++ b/modules/devices/faux-dev/python/src/dhcp_check.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Used to check if the DHCP server is functioning as expected""" import time diff --git a/net_orc/network/devices/faux-dev/python/src/dns_check.py b/modules/devices/faux-dev/python/src/dns_check.py similarity index 82% rename from net_orc/network/devices/faux-dev/python/src/dns_check.py rename to modules/devices/faux-dev/python/src/dns_check.py index 73a72e8c8..be9c58d43 100644 --- a/net_orc/network/devices/faux-dev/python/src/dns_check.py +++ b/modules/devices/faux-dev/python/src/dns_check.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Used to check if the DNS server is functioning as expected""" import logger diff --git a/net_orc/network/devices/faux-dev/python/src/gateway_check.py b/modules/devices/faux-dev/python/src/gateway_check.py similarity index 65% rename from net_orc/network/devices/faux-dev/python/src/gateway_check.py rename to modules/devices/faux-dev/python/src/gateway_check.py index 85fe35db0..a913993fc 100644 --- a/net_orc/network/devices/faux-dev/python/src/gateway_check.py +++ b/modules/devices/faux-dev/python/src/gateway_check.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Used to check if the Gateway server is functioning as expected""" import logger diff --git a/net_orc/network/devices/faux-dev/python/src/logger.py b/modules/devices/faux-dev/python/src/logger.py similarity index 67% rename from net_orc/network/devices/faux-dev/python/src/logger.py rename to modules/devices/faux-dev/python/src/logger.py index 97d7f935a..a727ad7bb 100644 --- a/net_orc/network/devices/faux-dev/python/src/logger.py +++ b/modules/devices/faux-dev/python/src/logger.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Sets up the logger to be used for the faux-device.""" import json diff --git a/net_orc/network/devices/faux-dev/python/src/ntp_check.py b/modules/devices/faux-dev/python/src/ntp_check.py similarity index 78% rename from net_orc/network/devices/faux-dev/python/src/ntp_check.py rename to modules/devices/faux-dev/python/src/ntp_check.py index ceef164c6..371e4464c 100644 --- a/net_orc/network/devices/faux-dev/python/src/ntp_check.py +++ b/modules/devices/faux-dev/python/src/ntp_check.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Used to check if the NTP server is functioning as expected""" import time import logger diff --git a/net_orc/network/devices/faux-dev/python/src/run.py b/modules/devices/faux-dev/python/src/run.py similarity index 84% rename from net_orc/network/devices/faux-dev/python/src/run.py rename to modules/devices/faux-dev/python/src/run.py index 062a1a643..8f9733eb4 100644 --- a/net_orc/network/devices/faux-dev/python/src/run.py +++ b/modules/devices/faux-dev/python/src/run.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Used to run all the various validator modules for the faux-device""" import argparse diff --git a/net_orc/network/devices/faux-dev/python/src/util.py b/modules/devices/faux-dev/python/src/util.py similarity index 60% rename from net_orc/network/devices/faux-dev/python/src/util.py rename to modules/devices/faux-dev/python/src/util.py index 6848206b4..920752217 100644 --- a/net_orc/network/devices/faux-dev/python/src/util.py +++ b/modules/devices/faux-dev/python/src/util.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Provides basic utilities for the faux-device.""" import subprocess import shlex diff --git a/net_orc/network/modules/base/base.Dockerfile b/modules/network/base/base.Dockerfile similarity index 74% rename from net_orc/network/modules/base/base.Dockerfile rename to modules/network/base/base.Dockerfile index 2400fd1c6..d14713c59 100644 --- a/net_orc/network/modules/base/base.Dockerfile +++ b/modules/network/base/base.Dockerfile @@ -1,17 +1,20 @@ # Image name: test-run/base FROM ubuntu:jammy +ARG MODULE_NAME=base +ARG MODULE_DIR=modules/network/$MODULE_NAME + # Install common software RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix -#Setup the base python requirements -COPY network/modules/base/python /testrun/python +# Setup the base python requirements +COPY $MODULE_DIR/python /testrun/python # Install all python requirements for the module RUN pip3 install -r /testrun/python/requirements.txt # Add the bin files -COPY network/modules/base/bin /testrun/bin +COPY $MODULE_DIR/bin /testrun/bin # Remove incorrect line endings RUN dos2unix /testrun/bin/* diff --git a/net_orc/network/modules/base/bin/capture b/modules/network/base/bin/capture similarity index 90% rename from net_orc/network/modules/base/bin/capture rename to modules/network/base/bin/capture index 8a8430feb..bc6c425e5 100644 --- a/net_orc/network/modules/base/bin/capture +++ b/modules/network/base/bin/capture @@ -23,7 +23,7 @@ fi # Create the output directory and start the capture mkdir -p $PCAP_DIR -chown $HOST_USER:$HOST_USER $PCAP_DIR +chown $HOST_USER $PCAP_DIR tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & #Small pause to let the capture to start diff --git a/net_orc/network/modules/base/bin/setup_binaries b/modules/network/base/bin/setup_binaries similarity index 100% rename from net_orc/network/modules/base/bin/setup_binaries rename to modules/network/base/bin/setup_binaries diff --git a/net_orc/network/modules/base/bin/start_grpc b/modules/network/base/bin/start_grpc similarity index 100% rename from net_orc/network/modules/base/bin/start_grpc rename to modules/network/base/bin/start_grpc diff --git a/net_orc/network/modules/base/bin/start_module b/modules/network/base/bin/start_module similarity index 100% rename from net_orc/network/modules/base/bin/start_module rename to modules/network/base/bin/start_module diff --git a/net_orc/network/modules/base/bin/start_network_service b/modules/network/base/bin/start_network_service similarity index 100% rename from net_orc/network/modules/base/bin/start_network_service rename to modules/network/base/bin/start_network_service diff --git a/net_orc/network/modules/base/bin/wait_for_interface b/modules/network/base/bin/wait_for_interface similarity index 100% rename from net_orc/network/modules/base/bin/wait_for_interface rename to modules/network/base/bin/wait_for_interface diff --git a/net_orc/network/modules/base/conf/module_config.json b/modules/network/base/conf/module_config.json similarity index 100% rename from net_orc/network/modules/base/conf/module_config.json rename to modules/network/base/conf/module_config.json diff --git a/net_orc/network/modules/base/python/requirements.txt b/modules/network/base/python/requirements.txt similarity index 100% rename from net_orc/network/modules/base/python/requirements.txt rename to modules/network/base/python/requirements.txt diff --git a/net_orc/network/modules/base/python/src/grpc/start_server.py b/modules/network/base/python/src/grpc/start_server.py similarity index 60% rename from net_orc/network/modules/base/python/src/grpc/start_server.py rename to modules/network/base/python/src/grpc/start_server.py index b4016c831..d372949e5 100644 --- a/net_orc/network/modules/base/python/src/grpc/start_server.py +++ b/modules/network/base/python/src/grpc/start_server.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Base class for starting the gRPC server for a network module.""" from concurrent import futures import grpc diff --git a/net_orc/network/modules/base/python/src/logger.py b/modules/network/base/python/src/logger.py similarity index 68% rename from net_orc/network/modules/base/python/src/logger.py rename to modules/network/base/python/src/logger.py index abec00f69..8893b1e8d 100644 --- a/net_orc/network/modules/base/python/src/logger.py +++ b/modules/network/base/python/src/logger.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Sets up the logger to be used for the network modules.""" import json import logging diff --git a/net_orc/network/modules/dhcp-1/bin/start_network_service b/modules/network/dhcp-1/bin/start_network_service similarity index 91% rename from net_orc/network/modules/dhcp-1/bin/start_network_service rename to modules/network/dhcp-1/bin/start_network_service index e8e0ad06c..a60806684 100644 --- a/net_orc/network/modules/dhcp-1/bin/start_network_service +++ b/modules/network/dhcp-1/bin/start_network_service @@ -21,8 +21,8 @@ mkdir /var/run/radvd #Create and set permissions on the log files touch $DHCP_LOG_FILE touch $RA_LOG_FILE -chown $HOST_USER:$HOST_USER $DHCP_LOG_FILE -chown $HOST_USER:$HOST_USER $RA_LOG_FILE +chown $HOST_USER $DHCP_LOG_FILE +chown $HOST_USER $RA_LOG_FILE #Move the config files to the correct location diff --git a/net_orc/network/modules/dhcp-1/conf/dhcpd.conf b/modules/network/dhcp-1/conf/dhcpd.conf similarity index 100% rename from net_orc/network/modules/dhcp-1/conf/dhcpd.conf rename to modules/network/dhcp-1/conf/dhcpd.conf diff --git a/net_orc/network/modules/dhcp-1/conf/module_config.json b/modules/network/dhcp-1/conf/module_config.json similarity index 100% rename from net_orc/network/modules/dhcp-1/conf/module_config.json rename to modules/network/dhcp-1/conf/module_config.json diff --git a/net_orc/network/modules/dhcp-1/conf/radvd.conf b/modules/network/dhcp-1/conf/radvd.conf similarity index 100% rename from net_orc/network/modules/dhcp-1/conf/radvd.conf rename to modules/network/dhcp-1/conf/radvd.conf diff --git a/net_orc/network/modules/dhcp-2/dhcp-2.Dockerfile b/modules/network/dhcp-1/dhcp-1.Dockerfile similarity index 56% rename from net_orc/network/modules/dhcp-2/dhcp-2.Dockerfile rename to modules/network/dhcp-1/dhcp-1.Dockerfile index 989992570..766f18c57 100644 --- a/net_orc/network/modules/dhcp-2/dhcp-2.Dockerfile +++ b/modules/network/dhcp-1/dhcp-1.Dockerfile @@ -1,14 +1,17 @@ # Image name: test-run/dhcp-primary FROM test-run/base:latest +ARG MODULE_NAME=dhcp-1 +ARG MODULE_DIR=modules/network/$MODULE_NAME + # Install dhcp server RUN apt-get install -y isc-dhcp-server radvd # Copy over all configuration files -COPY network/modules/dhcp-2/conf /testrun/conf +COPY $MODULE_DIR/conf /testrun/conf # Copy over all binary files -COPY network/modules/dhcp-2/bin /testrun/bin +COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY network/modules/dhcp-2/python /testrun/python +COPY $MODULE_DIR/python /testrun/python diff --git a/net_orc/network/modules/dhcp-1/python/src/grpc/__init__.py b/modules/network/dhcp-1/python/src/grpc/__init__.py similarity index 100% rename from net_orc/network/modules/dhcp-1/python/src/grpc/__init__.py rename to modules/network/dhcp-1/python/src/grpc/__init__.py diff --git a/net_orc/network/modules/dhcp-1/python/src/grpc/dhcp_config.py b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py similarity index 91% rename from net_orc/network/modules/dhcp-1/python/src/grpc/dhcp_config.py rename to modules/network/dhcp-1/python/src/grpc/dhcp_config.py index 23e1b4047..99d6bdebd 100644 --- a/net_orc/network/modules/dhcp-1/python/src/grpc/dhcp_config.py +++ b/modules/network/dhcp-1/python/src/grpc/dhcp_config.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Contains all the necessary classes to maintain the DHCP server's configuration""" import re diff --git a/net_orc/network/modules/dhcp-1/python/src/grpc/network_service.py b/modules/network/dhcp-1/python/src/grpc/network_service.py similarity index 70% rename from net_orc/network/modules/dhcp-1/python/src/grpc/network_service.py rename to modules/network/dhcp-1/python/src/grpc/network_service.py index 49732b362..64aab8a07 100644 --- a/net_orc/network/modules/dhcp-1/python/src/grpc/network_service.py +++ b/modules/network/dhcp-1/python/src/grpc/network_service.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """gRPC Network Service for the DHCP Server network module""" import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 diff --git a/net_orc/network/modules/dhcp-1/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-1/python/src/grpc/proto/grpc.proto similarity index 100% rename from net_orc/network/modules/dhcp-1/python/src/grpc/proto/grpc.proto rename to modules/network/dhcp-1/python/src/grpc/proto/grpc.proto diff --git a/net_orc/network/modules/dhcp-2/bin/start_network_service b/modules/network/dhcp-2/bin/start_network_service similarity index 91% rename from net_orc/network/modules/dhcp-2/bin/start_network_service rename to modules/network/dhcp-2/bin/start_network_service index d58174695..ad5ff09e7 100644 --- a/net_orc/network/modules/dhcp-2/bin/start_network_service +++ b/modules/network/dhcp-2/bin/start_network_service @@ -21,8 +21,8 @@ mkdir /var/run/radvd #Create and set permissions on the log files touch $DHCP_LOG_FILE touch $RA_LOG_FILE -chown $HOST_USER:$HOST_USER $DHCP_LOG_FILE -chown $HOST_USER:$HOST_USER $RA_LOG_FILE +chown $HOST_USER $DHCP_LOG_FILE +chown $HOST_USER $RA_LOG_FILE #Move the config files to the correct location diff --git a/net_orc/network/modules/dhcp-2/conf/dhcpd.conf b/modules/network/dhcp-2/conf/dhcpd.conf similarity index 100% rename from net_orc/network/modules/dhcp-2/conf/dhcpd.conf rename to modules/network/dhcp-2/conf/dhcpd.conf diff --git a/net_orc/network/modules/dhcp-2/conf/module_config.json b/modules/network/dhcp-2/conf/module_config.json similarity index 100% rename from net_orc/network/modules/dhcp-2/conf/module_config.json rename to modules/network/dhcp-2/conf/module_config.json diff --git a/net_orc/network/modules/dhcp-2/conf/radvd.conf b/modules/network/dhcp-2/conf/radvd.conf similarity index 100% rename from net_orc/network/modules/dhcp-2/conf/radvd.conf rename to modules/network/dhcp-2/conf/radvd.conf diff --git a/net_orc/network/modules/dhcp-1/dhcp-1.Dockerfile b/modules/network/dhcp-2/dhcp-2.Dockerfile similarity index 55% rename from net_orc/network/modules/dhcp-1/dhcp-1.Dockerfile rename to modules/network/dhcp-2/dhcp-2.Dockerfile index 99804e0e3..231d0c558 100644 --- a/net_orc/network/modules/dhcp-1/dhcp-1.Dockerfile +++ b/modules/network/dhcp-2/dhcp-2.Dockerfile @@ -1,14 +1,18 @@ # Image name: test-run/dhcp-primary FROM test-run/base:latest +ARG MODULE_NAME=dhcp-2 +ARG MODULE_DIR=modules/network/$MODULE_NAME + # Install dhcp server RUN apt-get install -y isc-dhcp-server radvd # Copy over all configuration files -COPY network/modules/dhcp-1/conf /testrun/conf +COPY $MODULE_DIR/conf /testrun/conf # Copy over all binary files -COPY network/modules/dhcp-1/bin /testrun/bin - +COPY $MODULE_DIR/bin /testrun/bin + # Copy over all python files -COPY network/modules/dhcp-1/python /testrun/python +COPY $MODULE_DIR/python /testrun/python + diff --git a/net_orc/network/modules/dhcp-2/python/src/grpc/__init__.py b/modules/network/dhcp-2/python/src/grpc/__init__.py similarity index 100% rename from net_orc/network/modules/dhcp-2/python/src/grpc/__init__.py rename to modules/network/dhcp-2/python/src/grpc/__init__.py diff --git a/net_orc/network/modules/dhcp-2/python/src/grpc/dhcp_config.py b/modules/network/dhcp-2/python/src/grpc/dhcp_config.py similarity index 91% rename from net_orc/network/modules/dhcp-2/python/src/grpc/dhcp_config.py rename to modules/network/dhcp-2/python/src/grpc/dhcp_config.py index 1d93c2d34..f6e79a2ec 100644 --- a/net_orc/network/modules/dhcp-2/python/src/grpc/dhcp_config.py +++ b/modules/network/dhcp-2/python/src/grpc/dhcp_config.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Contains all the necessary classes to maintain the DHCP server's configuration""" import re diff --git a/net_orc/network/modules/dhcp-2/python/src/grpc/network_service.py b/modules/network/dhcp-2/python/src/grpc/network_service.py similarity index 70% rename from net_orc/network/modules/dhcp-2/python/src/grpc/network_service.py rename to modules/network/dhcp-2/python/src/grpc/network_service.py index 49732b362..64aab8a07 100644 --- a/net_orc/network/modules/dhcp-2/python/src/grpc/network_service.py +++ b/modules/network/dhcp-2/python/src/grpc/network_service.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """gRPC Network Service for the DHCP Server network module""" import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 diff --git a/net_orc/network/modules/dhcp-2/python/src/grpc/proto/grpc.proto b/modules/network/dhcp-2/python/src/grpc/proto/grpc.proto similarity index 100% rename from net_orc/network/modules/dhcp-2/python/src/grpc/proto/grpc.proto rename to modules/network/dhcp-2/python/src/grpc/proto/grpc.proto diff --git a/net_orc/network/modules/dns/bin/start_network_service b/modules/network/dns/bin/start_network_service similarity index 100% rename from net_orc/network/modules/dns/bin/start_network_service rename to modules/network/dns/bin/start_network_service diff --git a/net_orc/network/modules/dns/conf/dnsmasq.conf b/modules/network/dns/conf/dnsmasq.conf similarity index 100% rename from net_orc/network/modules/dns/conf/dnsmasq.conf rename to modules/network/dns/conf/dnsmasq.conf diff --git a/net_orc/network/modules/dns/conf/module_config.json b/modules/network/dns/conf/module_config.json similarity index 100% rename from net_orc/network/modules/dns/conf/module_config.json rename to modules/network/dns/conf/module_config.json diff --git a/net_orc/network/modules/dns/dns.Dockerfile b/modules/network/dns/dns.Dockerfile similarity index 67% rename from net_orc/network/modules/dns/dns.Dockerfile rename to modules/network/dns/dns.Dockerfile index 84c1c7eb1..edfd4dd03 100644 --- a/net_orc/network/modules/dns/dns.Dockerfile +++ b/modules/network/dns/dns.Dockerfile @@ -1,6 +1,9 @@ # Image name: test-run/dns FROM test-run/base:latest +ARG MODULE_NAME=dns +ARG MODULE_DIR=modules/network/$MODULE_NAME + #Update and get all additional requirements not contained in the base image RUN apt-get update --fix-missing @@ -8,7 +11,7 @@ RUN apt-get update --fix-missing RUN apt-get install -y dnsmasq # Copy over all configuration files -COPY network/modules/dns/conf /testrun/conf +COPY $MODULE_DIR/conf /testrun/conf # Copy over all binary files -COPY network/modules/dns/bin /testrun/bin +COPY $MODULE_DIR/bin /testrun/bin diff --git a/net_orc/network/modules/gateway/bin/start_network_service b/modules/network/gateway/bin/start_network_service similarity index 100% rename from net_orc/network/modules/gateway/bin/start_network_service rename to modules/network/gateway/bin/start_network_service diff --git a/net_orc/network/modules/gateway/conf/module_config.json b/modules/network/gateway/conf/module_config.json similarity index 100% rename from net_orc/network/modules/gateway/conf/module_config.json rename to modules/network/gateway/conf/module_config.json diff --git a/net_orc/network/modules/gateway/gateway.Dockerfile b/modules/network/gateway/gateway.Dockerfile similarity index 59% rename from net_orc/network/modules/gateway/gateway.Dockerfile rename to modules/network/gateway/gateway.Dockerfile index b7085ebac..9bfa77dae 100644 --- a/net_orc/network/modules/gateway/gateway.Dockerfile +++ b/modules/network/gateway/gateway.Dockerfile @@ -1,11 +1,14 @@ # Image name: test-run/gateway FROM test-run/base:latest +ARG MODULE_NAME=gateway +ARG MODULE_DIR=modules/network/$MODULE_NAME + # Install required packages RUN apt-get install -y iptables isc-dhcp-client # Copy over all configuration files -COPY network/modules/gateway/conf /testrun/conf +COPY $MODULE_DIR/conf /testrun/conf # Copy over all binary files -COPY network/modules/gateway/bin /testrun/bin +COPY $MODULE_DIR/bin /testrun/bin diff --git a/net_orc/network/modules/ntp/bin/start_network_service b/modules/network/ntp/bin/start_network_service similarity index 82% rename from net_orc/network/modules/ntp/bin/start_network_service rename to modules/network/ntp/bin/start_network_service index 4c0c5dc74..b20cf8831 100644 --- a/net_orc/network/modules/ntp/bin/start_network_service +++ b/modules/network/ntp/bin/start_network_service @@ -7,7 +7,7 @@ echo Starting ntp #Create and set permissions on the log file touch $LOG_FILE -chown $HOST_USER:$HOST_USER $LOG_FILE +chown $HOST_USER $LOG_FILE #Start the NTP server python3 -u $PYTHON_SRC_DIR/ntp_server.py > $LOG_FILE diff --git a/net_orc/network/modules/ntp/conf/module_config.json b/modules/network/ntp/conf/module_config.json similarity index 100% rename from net_orc/network/modules/ntp/conf/module_config.json rename to modules/network/ntp/conf/module_config.json diff --git a/modules/network/ntp/ntp.Dockerfile b/modules/network/ntp/ntp.Dockerfile new file mode 100644 index 000000000..1add3178e --- /dev/null +++ b/modules/network/ntp/ntp.Dockerfile @@ -0,0 +1,16 @@ +# Image name: test-run/ntp +FROM test-run/base:latest + +ARG MODULE_NAME=ntp +ARG MODULE_DIR=modules/network/$MODULE_NAME + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python + +EXPOSE 123/udp diff --git a/net_orc/network/modules/ntp/python/src/ntp_server.py b/modules/network/ntp/python/src/ntp_server.py similarity index 90% rename from net_orc/network/modules/ntp/python/src/ntp_server.py rename to modules/network/ntp/python/src/ntp_server.py index 602585196..4eda2b13e 100644 --- a/net_orc/network/modules/ntp/python/src/ntp_server.py +++ b/modules/network/ntp/python/src/ntp_server.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """NTP Server""" import datetime import socket diff --git a/net_orc/network/modules/radius/bin/start_network_service b/modules/network/radius/bin/start_network_service similarity index 89% rename from net_orc/network/modules/radius/bin/start_network_service rename to modules/network/radius/bin/start_network_service index e27a828dd..399a90ae5 100644 --- a/net_orc/network/modules/radius/bin/start_network_service +++ b/modules/network/radius/bin/start_network_service @@ -15,6 +15,6 @@ python3 -u $PYTHON_SRC_DIR/authenticator.py & #Create and set permissions on the log file touch $LOG_FILE -chown $HOST_USER:$HOST_USER $LOG_FILE +chown $HOST_USER $LOG_FILE freeradius -f -X &> $LOG_FILE \ No newline at end of file diff --git a/net_orc/network/modules/radius/conf/ca.crt b/modules/network/radius/conf/ca.crt similarity index 100% rename from net_orc/network/modules/radius/conf/ca.crt rename to modules/network/radius/conf/ca.crt diff --git a/net_orc/network/modules/radius/conf/eap b/modules/network/radius/conf/eap similarity index 100% rename from net_orc/network/modules/radius/conf/eap rename to modules/network/radius/conf/eap diff --git a/net_orc/network/modules/radius/conf/module_config.json b/modules/network/radius/conf/module_config.json similarity index 100% rename from net_orc/network/modules/radius/conf/module_config.json rename to modules/network/radius/conf/module_config.json diff --git a/net_orc/network/modules/radius/python/requirements.txt b/modules/network/radius/python/requirements.txt similarity index 100% rename from net_orc/network/modules/radius/python/requirements.txt rename to modules/network/radius/python/requirements.txt diff --git a/net_orc/network/modules/radius/python/src/authenticator.py b/modules/network/radius/python/src/authenticator.py similarity index 71% rename from net_orc/network/modules/radius/python/src/authenticator.py rename to modules/network/radius/python/src/authenticator.py index 32f4ac221..0cca1921a 100644 --- a/net_orc/network/modules/radius/python/src/authenticator.py +++ b/modules/network/radius/python/src/authenticator.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Authenticator for the RADIUS Server""" from chewie.chewie import Chewie import logging diff --git a/net_orc/network/modules/radius/radius.Dockerfile b/modules/network/radius/radius.Dockerfile similarity index 74% rename from net_orc/network/modules/radius/radius.Dockerfile rename to modules/network/radius/radius.Dockerfile index a72313826..c44c5f0cc 100644 --- a/net_orc/network/modules/radius/radius.Dockerfile +++ b/modules/network/radius/radius.Dockerfile @@ -1,6 +1,9 @@ # Image name: test-run/radius FROM test-run/base:latest +ARG MODULE_NAME=radius +ARG MODULE_DIR=modules/network/$MODULE_NAME + # Install radius and git RUN apt-get update && apt-get install -y openssl freeradius git @@ -14,13 +17,13 @@ EXPOSE 1812/udp EXPOSE 1813/udp # Copy over all configuration files -COPY network/modules/radius/conf /testrun/conf +COPY $MODULE_DIR/conf /testrun/conf # Copy over all binary files -COPY network/modules/radius/bin /testrun/bin +COPY $MODULE_DIR/bin /testrun/bin # Copy over all python files -COPY network/modules/radius/python /testrun/python +COPY $MODULE_DIR/python /testrun/python # Install all python requirements for the module RUN pip3 install -r /testrun/python/requirements.txt \ No newline at end of file diff --git a/net_orc/network/modules/template/bin/start_network_service b/modules/network/template/bin/start_network_service similarity index 100% rename from net_orc/network/modules/template/bin/start_network_service rename to modules/network/template/bin/start_network_service diff --git a/net_orc/network/modules/template/conf/module_config.json b/modules/network/template/conf/module_config.json similarity index 91% rename from net_orc/network/modules/template/conf/module_config.json rename to modules/network/template/conf/module_config.json index c767c9ad6..e702e1804 100644 --- a/net_orc/network/modules/template/conf/module_config.json +++ b/modules/network/template/conf/module_config.json @@ -15,6 +15,7 @@ }, "docker": { "enable_container": false, + "template":true, "depends_on": "base", "mounts": [ { diff --git a/modules/network/template/python/src/template_main.py b/modules/network/template/python/src/template_main.py new file mode 100644 index 000000000..ddf83e2c4 --- /dev/null +++ b/modules/network/template/python/src/template_main.py @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python code for the template module.""" + +if __name__ == "__main__": + print("Template main") diff --git a/modules/network/template/template.Dockerfile b/modules/network/template/template.Dockerfile new file mode 100644 index 000000000..9efbfb230 --- /dev/null +++ b/modules/network/template/template.Dockerfile @@ -0,0 +1,14 @@ +# Image name: test-run/template +FROM test-run/base:latest + +ARG MODULE_NAME=template +ARG MODULE_DIR=modules/network/$MODULE_NAME + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/base/base.Dockerfile b/modules/test/base/base.Dockerfile similarity index 74% rename from test_orc/modules/base/base.Dockerfile rename to modules/test/base/base.Dockerfile index a508caef7..b8398eae9 100644 --- a/test_orc/modules/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -1,17 +1,20 @@ # Image name: test-run/base-test FROM ubuntu:jammy +ARG MODULE_NAME=base +ARG MODULE_DIR=modules/test/$MODULE_NAME + # Install common software RUN apt-get update && apt-get install -y net-tools iputils-ping tcpdump iproute2 jq python3 python3-pip dos2unix nmap --fix-missing # Setup the base python requirements -COPY modules/base/python /testrun/python +COPY $MODULE_DIR/python /testrun/python # Install all python requirements for the module RUN pip3 install -r /testrun/python/requirements.txt -# Add the bin files -COPY modules/base/bin /testrun/bin +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin # Remove incorrect line endings RUN dos2unix /testrun/bin/* diff --git a/test_orc/modules/base/bin/capture b/modules/test/base/bin/capture similarity index 88% rename from test_orc/modules/base/bin/capture rename to modules/test/base/bin/capture index facb6acf7..45cfcd42f 100644 --- a/test_orc/modules/base/bin/capture +++ b/modules/test/base/bin/capture @@ -12,7 +12,7 @@ INTERFACE=$2 # Create the output directory and start the capture mkdir -p $PCAP_DIR -chown $HOST_USER:$HOST_USER $PCAP_DIR +chown $HOST_USER $PCAP_DIR tcpdump -i $INTERFACE -w $PCAP_DIR/$PCAP_FILE -Z $HOST_USER & # Small pause to let the capture to start diff --git a/test_orc/modules/base/bin/get_ipv4_addr b/modules/test/base/bin/get_ipv4_addr similarity index 100% rename from test_orc/modules/base/bin/get_ipv4_addr rename to modules/test/base/bin/get_ipv4_addr diff --git a/test_orc/modules/base/bin/setup_binaries b/modules/test/base/bin/setup_binaries similarity index 100% rename from test_orc/modules/base/bin/setup_binaries rename to modules/test/base/bin/setup_binaries diff --git a/test_orc/modules/base/bin/start_grpc b/modules/test/base/bin/start_grpc similarity index 100% rename from test_orc/modules/base/bin/start_grpc rename to modules/test/base/bin/start_grpc diff --git a/test_orc/modules/base/bin/start_module b/modules/test/base/bin/start_module similarity index 91% rename from test_orc/modules/base/bin/start_module rename to modules/test/base/bin/start_module index 6adc53f58..3e4737d8b 100644 --- a/test_orc/modules/base/bin/start_module +++ b/modules/test/base/bin/start_module @@ -1,5 +1,8 @@ #!/bin/bash +# Define the local mount point to store local files to +OUTPUT_DIR="/runtime/output" + # Directory where all binaries will be loaded BIN_DIR="/testrun/bin" @@ -11,6 +14,9 @@ IFACE=veth0 # HOST_USER mapped in via docker container environemnt variables useradd $HOST_USER +# Set permissions on the output files +chown -R $HOST_USER $OUTPUT_DIR + # Enable IPv6 for all containers sysctl net.ipv6.conf.all.disable_ipv6=0 sysctl -p diff --git a/test_orc/modules/base/bin/wait_for_interface b/modules/test/base/bin/wait_for_interface similarity index 100% rename from test_orc/modules/base/bin/wait_for_interface rename to modules/test/base/bin/wait_for_interface diff --git a/test_orc/modules/base/conf/module_config.json b/modules/test/base/conf/module_config.json similarity index 100% rename from test_orc/modules/base/conf/module_config.json rename to modules/test/base/conf/module_config.json diff --git a/test_orc/modules/base/python/requirements.txt b/modules/test/base/python/requirements.txt similarity index 100% rename from test_orc/modules/base/python/requirements.txt rename to modules/test/base/python/requirements.txt diff --git a/test_orc/modules/base/python/src/grpc/start_server.py b/modules/test/base/python/src/grpc/start_server.py similarity index 60% rename from test_orc/modules/base/python/src/grpc/start_server.py rename to modules/test/base/python/src/grpc/start_server.py index b4016c831..d372949e5 100644 --- a/test_orc/modules/base/python/src/grpc/start_server.py +++ b/modules/test/base/python/src/grpc/start_server.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Base class for starting the gRPC server for a network module.""" from concurrent import futures import grpc diff --git a/test_orc/modules/base/python/src/logger.py b/modules/test/base/python/src/logger.py similarity index 67% rename from test_orc/modules/base/python/src/logger.py rename to modules/test/base/python/src/logger.py index 42124beea..64594c7b3 100644 --- a/test_orc/modules/base/python/src/logger.py +++ b/modules/test/base/python/src/logger.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Sets up the logger to be used for the test modules.""" import json import logging diff --git a/test_orc/modules/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py similarity index 80% rename from test_orc/modules/base/python/src/test_module.py rename to modules/test/base/python/src/test_module.py index 8e10a3637..f29668bb2 100644 --- a/test_orc/modules/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Base class for all core test module functions""" import json import logger @@ -81,7 +95,12 @@ def run_tests(self): else: LOGGER.info('Test ' + test['name'] + ' disabled. Skipping') if result is not None: - test['result'] = 'compliant' if result else 'non-compliant' + success = None + if isinstance(result,bool): + test['result'] = 'compliant' if result else 'non-compliant' + else: + test['result'] = 'compliant' if result[0] else 'non-compliant' + test['result_details'] = result[1] else: test['result'] = 'skipped' test['end'] = datetime.now().isoformat() diff --git a/test_orc/modules/base/python/src/util.py b/modules/test/base/python/src/util.py similarity index 61% rename from test_orc/modules/base/python/src/util.py rename to modules/test/base/python/src/util.py index d387db796..0f54c4298 100644 --- a/test_orc/modules/base/python/src/util.py +++ b/modules/test/base/python/src/util.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Provides basic utilities for a test module.""" import subprocess import shlex diff --git a/modules/test/baseline/baseline.Dockerfile b/modules/test/baseline/baseline.Dockerfile new file mode 100644 index 000000000..c2b32e7b7 --- /dev/null +++ b/modules/test/baseline/baseline.Dockerfile @@ -0,0 +1,14 @@ +# Image name: test-run/baseline-test +FROM test-run/base-test:latest + +ARG MODULE_NAME=baseline +ARG MODULE_DIR=modules/test/$MODULE_NAME + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/baseline/bin/start_test_module b/modules/test/baseline/bin/start_test_module similarity index 90% rename from test_orc/modules/baseline/bin/start_test_module rename to modules/test/baseline/bin/start_test_module index 2938eb0f8..a09349cf9 100644 --- a/test_orc/modules/baseline/bin/start_test_module +++ b/modules/test/baseline/bin/start_test_module @@ -31,8 +31,8 @@ LOG_FILE=/runtime/output/$MODULE_NAME.log RESULT_FILE=/runtime/output/$MODULE_NAME-result.json touch $LOG_FILE touch $RESULT_FILE -chown $HOST_USER:$HOST_USER $LOG_FILE -chown $HOST_USER:$HOST_USER $RESULT_FILE +chown $HOST_USER $LOG_FILE +chown $HOST_USER $RESULT_FILE # Run the python scrip that will execute the tests for this module # -u flag allows python print statements diff --git a/test_orc/modules/baseline/conf/module_config.json b/modules/test/baseline/conf/module_config.json similarity index 100% rename from test_orc/modules/baseline/conf/module_config.json rename to modules/test/baseline/conf/module_config.json diff --git a/test_orc/modules/baseline/python/src/baseline_module.py b/modules/test/baseline/python/src/baseline_module.py similarity index 53% rename from test_orc/modules/baseline/python/src/baseline_module.py rename to modules/test/baseline/python/src/baseline_module.py index 083123436..22555d369 100644 --- a/test_orc/modules/baseline/python/src/baseline_module.py +++ b/modules/test/baseline/python/src/baseline_module.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Baseline test module""" from test_module import TestModule diff --git a/test_orc/modules/baseline/python/src/run.py b/modules/test/baseline/python/src/run.py similarity index 69% rename from test_orc/modules/baseline/python/src/run.py rename to modules/test/baseline/python/src/run.py index 1892ed8ae..42eccbef4 100644 --- a/test_orc/modules/baseline/python/src/run.py +++ b/modules/test/baseline/python/src/run.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Run Baseline module""" import argparse import signal diff --git a/modules/test/conn/bin/start_test_module b/modules/test/conn/bin/start_test_module new file mode 100644 index 000000000..8290c0764 --- /dev/null +++ b/modules/test/conn/bin/start_test_module @@ -0,0 +1,39 @@ +#!/bin/bash + +# Setup and start the connection test module + +# Define where the python source files are located +PYTHON_SRC_DIR=/testrun/python/src + +# Fetch module name +MODULE_NAME=$1 + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Allow a user to define an interface by passing it into this script +DEFINED_IFACE=$2 + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No interface defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +# Create and set permissions on the log files +LOG_FILE=/runtime/output/$MODULE_NAME.log +RESULT_FILE=/runtime/output/$MODULE_NAME-result.json +touch $LOG_FILE +touch $RESULT_FILE +chown $HOST_USER $LOG_FILE +chown $HOST_USER $RESULT_FILE + +# Run the python scrip that will execute the tests for this module +# -u flag allows python print statements +# to be logged by docker by running unbuffered +python3 -u $PYTHON_SRC_DIR/run.py "-m $MODULE_NAME" + +echo Module has finished \ No newline at end of file diff --git a/modules/test/conn/conf/module_config.json b/modules/test/conn/conf/module_config.json new file mode 100644 index 000000000..0f599c5d3 --- /dev/null +++ b/modules/test/conn/conf/module_config.json @@ -0,0 +1,37 @@ +{ + "config": { + "meta": { + "name": "connection", + "display_name": "Connection", + "description": "Connection tests" + }, + "network": true, + "docker": { + "depends_on": "base", + "enable_container": true, + "timeout": 30 + }, + "tests": [ + { + "name": "connection.mac_address", + "description": "Check and note device physical address.", + "expected_behavior": "N/A" + }, + { + "name": "connection.mac_oui", + "description": "The device under test hs a MAC address prefix that is registered against a known manufacturer.", + "expected_behavior": "The MAC address prefix is registered in the IEEE Organizationally Unique Identifier database." + }, + { + "name": "connection.single_ip", + "description": "The network switch port connected to the device reports only one IP address for the device under test.", + "expected_behavior": "The device under test does not behave as a network switch and only requets one IP address. This test is to avoid that devices implement network switches that allow connecting strings of daisy chained devices to one single network port, as this would not make 802.1x port based authentication possible." + }, + { + "name": "connection.target_ping", + "description": "The device under test responds to an ICMP echo (ping) request.", + "expected_behavior": "The device under test responds to an ICMP echo (ping) request." + } + ] + } +} \ No newline at end of file diff --git a/modules/test/conn/conn.Dockerfile b/modules/test/conn/conn.Dockerfile new file mode 100644 index 000000000..2526b0046 --- /dev/null +++ b/modules/test/conn/conn.Dockerfile @@ -0,0 +1,26 @@ +# Image name: test-run/conn-test +FROM test-run/base-test:latest + +ARG MODULE_NAME=conn +ARG MODULE_DIR=modules/test/$MODULE_NAME + +# Install all necessary packages +RUN apt-get install -y wget + +#Update the oui.txt file from ieee +RUN wget http://standards-oui.ieee.org/oui.txt -P /usr/local/etc/ + +#Load the requirements file +COPY $MODULE_DIR/python/requirements.txt /testrun/python + +#Install all python requirements for the module +RUN pip3 install -r /testrun/python/requirements.txt + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python diff --git a/modules/test/conn/python/requirements.txt b/modules/test/conn/python/requirements.txt new file mode 100644 index 000000000..93b351f44 --- /dev/null +++ b/modules/test/conn/python/requirements.txt @@ -0,0 +1 @@ +scapy \ No newline at end of file diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py new file mode 100644 index 000000000..196c335d8 --- /dev/null +++ b/modules/test/conn/python/src/connection_module.py @@ -0,0 +1,117 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Connection test module""" +import util +import sys +from scapy.all import * +from test_module import TestModule + +LOG_NAME = "test_connection" +LOGGER = None +OUI_FILE="/usr/local/etc/oui.txt" +DHCP_SERVER_CAPTURE_FILE = '/runtime/network/dhcp-1.pcap' +STARTUP_CAPTURE_FILE = '/runtime/device/startup.pcap' +MONITOR_CAPTURE_FILE = '/runtime/device/monitor.pcap' + + +class ConnectionModule(TestModule): + """Connection Test module""" + + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + global LOGGER + LOGGER = self._get_logger() + + def _connection_mac_address(self): + LOGGER.info("Running connection.mac_address") + if self._device_mac is not None: + LOGGER.info("MAC address found: " + self._device_mac) + return True, "MAC address found: " + self._device_mac + else: + LOGGER.info("No MAC address found: " + self._device_mac) + return False, "No MAC address found." + + def _connection_mac_oui(self): + LOGGER.info("Running connection.mac_oui") + manufacturer = self._get_oui_manufacturer(self._device_mac) + if manufacturer is not None: + LOGGER.info("OUI Manufacturer found: " + manufacturer) + return True, "OUI Manufacturer found: " + manufacturer + else: + LOGGER.info("No OUI Manufacturer found for: " + self._device_mac) + return False, "No OUI Manufacturer found for: " + self._device_mac + + def _connection_single_ip(self): + LOGGER.info("Running connection.single_ip") + + result = None + if self._device_mac is None: + LOGGER.info("No MAC address found: ") + return result, "No MAC address found." + + # Read all the pcap files containing DHCP packet information + packets = rdpcap(DHCP_SERVER_CAPTURE_FILE) + packets.append(rdpcap(STARTUP_CAPTURE_FILE)) + packets.append(rdpcap(MONITOR_CAPTURE_FILE)) + + # Extract MAC addresses from DHCP packets + mac_addresses = set() + LOGGER.info("Inspecting: " + str(len(packets)) + " packets") + for packet in packets: + # Option[1] = message-type, option 3 = DHCPREQUEST + if DHCP in packet and packet[DHCP].options[0][1] == 3: + mac_address = packet[Ether].src + mac_addresses.add(mac_address.upper()) + + # Check if the device mac address is in the list of DHCPREQUESTs + result = self._device_mac.upper() in mac_addresses + LOGGER.info("DHCPREQUEST detected from device: " + str(result)) + + # Check the unique MAC addresses to see if they match the device + for mac_address in mac_addresses: + LOGGER.info("DHCPREQUEST from MAC address: " + mac_address) + result &= self._device_mac.upper() == mac_address + return result + + + def _connection_target_ping(self): + LOGGER.info("Running connection.target_ping") + + # If the ipv4 address wasn't resolved yet, try again + if self._device_ipv4_addr is None: + self._device_ipv4_addr = self._get_device_ipv4(self) + + if self._device_ipv4_addr is None: + LOGGER.error("No device IP could be resolved") + sys.exit(1) + else: + return self._ping(self._device_ipv4_addr) + + def _get_oui_manufacturer(self,mac_address): + # Do some quick fixes on the format of the mac_address + # to match the oui file pattern + mac_address = mac_address.replace(":","-").upper() + with open(OUI_FILE, "r") as file: + for line in file: + if mac_address.startswith(line[:8]): + start = line.index("(hex)") + len("(hex)") + return line[start:].strip() # Extract the company name + return None + + def _ping(self, host): + cmd = 'ping -c 1 ' + str(host) + success = util.run_command(cmd, output=False) + return success + \ No newline at end of file diff --git a/modules/test/conn/python/src/run.py b/modules/test/conn/python/src/run.py new file mode 100644 index 000000000..5165b58c6 --- /dev/null +++ b/modules/test/conn/python/src/run.py @@ -0,0 +1,68 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run NMAP test module""" +import argparse +import signal +import sys +import logger + +from connection_module import ConnectionModule + +LOGGER = logger.get_logger('connection_module') + + +class ConnectionModuleRunner: + """Run the Connection module tests.""" + + def __init__(self, module): + + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) + + LOGGER.info('Starting connection module') + + self._test_module = ConnectionModule(module) + self._test_module.run_tests() + + def _handler(self, signum): + LOGGER.debug('SigtermEnum: ' + str(signal.SIGTERM)) + LOGGER.debug('Exit signal received: ' + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info('Exit signal received. Stopping connection test module...') + LOGGER.info('Test module stopped') + sys.exit(1) + + +def run(): + parser = argparse.ArgumentParser( + description='Connection Module Help', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument( + '-m', + '--module', + help='Define the module name to be used to create the log file') + + args = parser.parse_args() + + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + ConnectionModuleRunner(args.module.strip()) + + +if __name__ == '__main__': + run() diff --git a/test_orc/modules/dns/bin/start_test_module b/modules/test/dns/bin/start_test_module similarity index 90% rename from test_orc/modules/dns/bin/start_test_module rename to modules/test/dns/bin/start_test_module index 2938eb0f8..a09349cf9 100644 --- a/test_orc/modules/dns/bin/start_test_module +++ b/modules/test/dns/bin/start_test_module @@ -31,8 +31,8 @@ LOG_FILE=/runtime/output/$MODULE_NAME.log RESULT_FILE=/runtime/output/$MODULE_NAME-result.json touch $LOG_FILE touch $RESULT_FILE -chown $HOST_USER:$HOST_USER $LOG_FILE -chown $HOST_USER:$HOST_USER $RESULT_FILE +chown $HOST_USER $LOG_FILE +chown $HOST_USER $RESULT_FILE # Run the python scrip that will execute the tests for this module # -u flag allows python print statements diff --git a/test_orc/modules/dns/conf/module_config.json b/modules/test/dns/conf/module_config.json similarity index 80% rename from test_orc/modules/dns/conf/module_config.json rename to modules/test/dns/conf/module_config.json index b8ff36c97..177537b69 100644 --- a/test_orc/modules/dns/conf/module_config.json +++ b/modules/test/dns/conf/module_config.json @@ -21,6 +21,10 @@ "name": "dns.network.from_dhcp", "description": "Verify the device allows for a DNS server to be entered automatically", "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server" + }, + { + "name": "dns.mdns", + "description": "If the device has MDNS (or any kind of IP multicast), can it be disabled" } ] } diff --git a/modules/test/dns/dns.Dockerfile b/modules/test/dns/dns.Dockerfile new file mode 100644 index 000000000..f831d0e2b --- /dev/null +++ b/modules/test/dns/dns.Dockerfile @@ -0,0 +1,14 @@ +# Image name: test-run/conn-test +FROM test-run/base-test:latest + +ARG MODULE_NAME=dns +ARG MODULE_DIR=modules/test/$MODULE_NAME + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/dns/python/src/dns_module.py b/modules/test/dns/python/src/dns_module.py similarity index 50% rename from test_orc/modules/dns/python/src/dns_module.py rename to modules/test/dns/python/src/dns_module.py index 58ce48123..8d32d4dfb 100644 --- a/test_orc/modules/dns/python/src/dns_module.py +++ b/modules/test/dns/python/src/dns_module.py @@ -1,9 +1,25 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """DNS test module""" import subprocess from test_module import TestModule LOG_NAME = 'test_dns' -CAPTURE_FILE = '/runtime/network/dns.pcap' +DNS_SERVER_CAPTURE_FILE = '/runtime/network/dns.pcap' +STARTUP_CAPTURE_FILE = '/runtime/device/startup.pcap' +MONITOR_CAPTURE_FILE = '/runtime/device/monitor.pcap' LOGGER = None @@ -17,14 +33,24 @@ def __init__(self, module): LOGGER = self._get_logger() def _check_dns_traffic(self, tcpdump_filter): - to_dns = self._exec_tcpdump(tcpdump_filter) - num_query_dns = len(to_dns) + dns_server_queries = self._exec_tcpdump(tcpdump_filter,DNS_SERVER_CAPTURE_FILE) + LOGGER.info('DNS Server queries found: ' + str(len(dns_server_queries))) + + dns_startup_queries = self._exec_tcpdump(tcpdump_filter,STARTUP_CAPTURE_FILE) + LOGGER.info('Startup DNS queries found: ' + str(len(dns_startup_queries))) + + dns_monitor_queries = self._exec_tcpdump(tcpdump_filter,MONITOR_CAPTURE_FILE) + LOGGER.info('Monitor DNS queries found: ' + str(len(dns_monitor_queries))) + + num_query_dns = len(dns_server_queries) + len(dns_startup_queries) + len(dns_monitor_queries) + LOGGER.info('DNS queries found: ' + str(num_query_dns)) - dns_traffic_detected = len(to_dns) > 0 + dns_traffic_detected = num_query_dns > 0 LOGGER.info('DNS traffic detected: ' + str(dns_traffic_detected)) return dns_traffic_detected def _dns_network_from_dhcp(self): + LOGGER.info("Running dns.network.from_dhcp") LOGGER.info('Checking DNS traffic for configured DHCP DNS server: ' + self._dns_server) @@ -39,6 +65,7 @@ def _dns_network_from_dhcp(self): return result def _dns_network_from_device(self): + LOGGER.info("Running dns.network.from_device") LOGGER.info('Checking DNS traffic from device: ' + self._device_mac) # Check if the device DNS traffic is to appropriate server @@ -49,7 +76,19 @@ def _dns_network_from_device(self): LOGGER.info('DNS traffic detected from device: ' + str(result)) return result - def _exec_tcpdump(self, tcpdump_filter): + def _dns_mdns(self): + LOGGER.info("Running dns.mdns") + + # Check if the device sends any MDNS traffic + tcpdump_filter = f'udp port 5353 and ether src {self._device_mac}' + + result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) + + LOGGER.info('MDNS traffic detected from device: ' + str(result)) + return not result + + + def _exec_tcpdump(self, tcpdump_filter, capture_file): """ Args tcpdump_filter: Filter to pass onto tcpdump file @@ -57,7 +96,7 @@ def _exec_tcpdump(self, tcpdump_filter): Returns List of packets matching the filter """ - command = f'tcpdump -tttt -n -r {CAPTURE_FILE} {tcpdump_filter}' + command = f'tcpdump -tttt -n -r {capture_file} {tcpdump_filter}' LOGGER.debug('tcpdump command: ' + command) diff --git a/test_orc/modules/dns/python/src/run.py b/modules/test/dns/python/src/run.py similarity index 64% rename from test_orc/modules/dns/python/src/run.py rename to modules/test/dns/python/src/run.py index 4cd991804..4803f63cd 100644 --- a/test_orc/modules/dns/python/src/run.py +++ b/modules/test/dns/python/src/run.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Run DNS test module""" import argparse import signal @@ -6,7 +20,7 @@ from dns_module import DNSModule -LOG_NAME = "dns_module" +LOG_NAME = "dns_runner" LOGGER = logger.get_logger(LOG_NAME) RUNTIME = 1500 @@ -21,12 +35,12 @@ def __init__(self, module): signal.signal(signal.SIGQUIT, self._handler) self.add_logger(module) - LOGGER.info("Starting DNS Test Module") + LOGGER.info("Starting DNS test module") self._test_module = DNSModule(module) self._test_module.run_tests() - LOGGER.info("DNS Test Module Finished") + LOGGER.info("DNS test module finished") def add_logger(self, module): global LOGGER @@ -43,7 +57,7 @@ def _handler(self, signum): def run(): parser = argparse.ArgumentParser( - description="Test Module DNS", + description="DNS Module Help", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( diff --git a/test_orc/modules/nmap/bin/start_test_module b/modules/test/nmap/bin/start_test_module similarity index 93% rename from test_orc/modules/nmap/bin/start_test_module rename to modules/test/nmap/bin/start_test_module index 4bb7e9f96..333566342 100644 --- a/test_orc/modules/nmap/bin/start_test_module +++ b/modules/test/nmap/bin/start_test_module @@ -31,8 +31,8 @@ LOG_FILE=/runtime/output/$MODULE_NAME.log RESULT_FILE=/runtime/output/$MODULE_NAME-result.json touch $LOG_FILE touch $RESULT_FILE -chown $HOST_USER:$HOST_USER $LOG_FILE -chown $HOST_USER:$HOST_USER $RESULT_FILE +chown $HOST_USER $LOG_FILE +chown $HOST_USER $RESULT_FILE # Run the python scrip that will execute the tests for this module # -u flag allows python print statements diff --git a/test_orc/modules/nmap/conf/module_config.json b/modules/test/nmap/conf/module_config.json similarity index 96% rename from test_orc/modules/nmap/conf/module_config.json rename to modules/test/nmap/conf/module_config.json index 5449327a1..aafde4c03 100644 --- a/test_orc/modules/nmap/conf/module_config.json +++ b/modules/test/nmap/conf/module_config.json @@ -34,7 +34,8 @@ "tcp_ports": { "22": { "allowed": true, - "description": "Secure Shell (SSH) server" + "description": "Secure Shell (SSH) server", + "version": "2.0" } }, "description": "Check TELNET port 23 is disabled and TELNET is not running on any port", diff --git a/modules/test/nmap/nmap.Dockerfile b/modules/test/nmap/nmap.Dockerfile new file mode 100644 index 000000000..c1a2f96ce --- /dev/null +++ b/modules/test/nmap/nmap.Dockerfile @@ -0,0 +1,20 @@ +# Image name: test-run/nmap-test +FROM test-run/base-test:latest + +ARG MODULE_NAME=nmap +ARG MODULE_DIR=modules/test/$MODULE_NAME + +#Load the requirements file +COPY $MODULE_DIR/python/requirements.txt /testrun/python + +#Install all python requirements for the module +RUN pip3 install -r /testrun/python/requirements.txt + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python \ No newline at end of file diff --git a/modules/test/nmap/python/requirements.txt b/modules/test/nmap/python/requirements.txt new file mode 100644 index 000000000..42669b12c --- /dev/null +++ b/modules/test/nmap/python/requirements.txt @@ -0,0 +1 @@ +xmltodict \ No newline at end of file diff --git a/modules/test/nmap/python/src/nmap_module.py b/modules/test/nmap/python/src/nmap_module.py new file mode 100644 index 000000000..ea013f413 --- /dev/null +++ b/modules/test/nmap/python/src/nmap_module.py @@ -0,0 +1,381 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NMAP test module""" +import time +import util +import json +import threading +import xmltodict +import re +from test_module import TestModule + +LOG_NAME = "test_nmap" +LOGGER = None + + +class NmapModule(TestModule): + """NMAP Test module""" + + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + self._unallowed_ports = [] + self._scan_tcp_results = None + self._udp_tcp_results = None + self._script_scan_results = None + global LOGGER + LOGGER = self._get_logger() + + + def _security_nmap_ports(self, config): + LOGGER.info("Running security.nmap.ports test") + + # Delete the enabled key from the config if it exists + # to prevent it being treated as a test key + if "enabled" in config: + del config["enabled"] + + if self._device_ipv4_addr is not None: + # Run the monitor method asynchronously to keep this method non-blocking + self._tcp_scan_thread = threading.Thread(target=self._scan_tcp_ports, + args=(config, )) + self._udp_scan_thread = threading.Thread(target=self._scan_udp_ports, + args=(config, )) + self._script_scan_thread = threading.Thread(target=self._scan_scripts, + args=(config, )) + + self._tcp_scan_thread.daemon = True + self._udp_scan_thread.daemon = True + self._script_scan_thread.daemon = True + + self._tcp_scan_thread.start() + self._udp_scan_thread.start() + self._script_scan_thread.start() + + while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive( + ) or self._script_scan_thread.is_alive(): + time.sleep(1) + + LOGGER.debug("TCP scan results: " + str(self._scan_tcp_results)) + LOGGER.debug("UDP scan results: " + str(self._scan_udp_results)) + LOGGER.debug("Service scan results: " + str(self._script_scan_results)) + self._process_port_results(tests=config) + LOGGER.info("Unallowed Ports Detected: " + str(self._unallowed_ports)) + self._check_unallowed_port(self._unallowed_ports,config) + LOGGER.info("Unallowed Ports: " + str(self._unallowed_ports)) + return len(self._unallowed_ports) == 0 + else: + LOGGER.info("Device ip address not resolved, skipping") + return None + + def _process_port_results(self, tests): + scan_results = {} + if self._scan_tcp_results is not None: + scan_results.update(self._scan_tcp_results) + if self._scan_udp_results is not None: + scan_results.update(self._scan_udp_results) + if self._script_scan_results is not None: + scan_results.update(self._script_scan_results) + + self._check_unknown_ports(tests=tests,scan_results=scan_results) + + for test in tests: + LOGGER.info("Checking scan results for test: " + str(test)) + self._check_scan_results(test_config=tests[test],scan_results=scan_results) + + def _check_unknown_ports(self,tests,scan_results): + """ Check if any of the open ports detected are not defined + in the test configurations. If an open port is detected + without a configuration associated with it, the default behavior + is to mark it as an unallowed port. + """ + known_ports = [] + for test in tests: + if "tcp_ports" in tests[test]: + for port in tests[test]['tcp_ports']: + known_ports.append(port) + if "udp_ports" in tests[test]: + for port in tests[test]['udp_ports']: + known_ports.append(port) + + for port_result in scan_results: + if not port_result in known_ports: + LOGGER.info("Unknown port detected: " + port_result) + unallowed_port = {'port':port_result, + 'service':scan_results[port_result]['service'], + 'tcp_udp':scan_results[port_result]['tcp_udp']} + #self._unallowed_ports.append(unallowed_port) + self._add_unknown_ports(tests,unallowed_port) + + def _add_unknown_ports(self,tests,unallowed_port): + known_service = False + result = {'description':"Undefined port",'allowed':False} + if unallowed_port['tcp_udp'] == 'tcp': + port_style = 'tcp_ports' + elif unallowed_port['tcp_udp'] == 'udp': + port_style = 'udp_ports' + + LOGGER.info("Unknown Port Service: " + unallowed_port['service']) + for test in tests: + LOGGER.debug("Checking for known service: " + test) + # Create a regular expression pattern to match the variable at the + # end of the string + port_service = r"\b" + re.escape(unallowed_port['service']) + r"\b$" + service_match = re.search(port_service, test) + if service_match: + LOGGER.info("Service Matched: " + test) + known_service=True + for test_port in tests[test][port_style]: + if "version" in tests[test][port_style][test_port]: + result['version'] = tests[test][port_style][test_port]['version'] + if "description" in tests[test][port_style][test_port]: + result['description'] = tests[test][port_style][test_port]['description'] + result['inherited_from'] = test_port + if tests[test][port_style][test_port]['allowed']: + result['allowed'] = True + break + tests[test][port_style][unallowed_port['port']]=result + break + + if not known_service: + service_name = "security.services.unknown." + str(unallowed_port['port']) + unknown_service = {port_style:{unallowed_port['port']:result}} + tests[service_name]=unknown_service + + def _check_scan_results(self, test_config,scan_results): + port_config = {} + if "tcp_ports" in test_config: + port_config.update(test_config["tcp_ports"]) + elif "udp_ports" in test_config: + port_config.update(test_config["udp_ports"]) + + if port_config is not None: + for port, config in port_config.items(): + result = None + LOGGER.info("Checking port: " + str(port)) + LOGGER.debug("Port config: " + str(config)) + if port in scan_results: + if scan_results[port]["state"] == "open": + if not config["allowed"]: + LOGGER.info("Unallowed port open") + self._unallowed_ports.append( + {"port":str(port), + "service":str(scan_results[port]["service"]), + 'tcp_udp':scan_results[port]['tcp_udp']} + ) + result = False + else: + LOGGER.info("Allowed port open") + if "version" in config and "version" in scan_results[port]: + version_check = self._check_version(scan_results[port]["service"], + scan_results[port]["version"],config["version"]) + if version_check is not None: + result = version_check + else: + result = True + else: + result = True + else: + LOGGER.info("Port is closed") + result = True + else: + LOGGER.info("Port not detected, closed") + result = True + + if result is not None: + config["result"] = "compliant" if result else "non-compliant" + else: + config["result"] = "skipped" + + def _check_unallowed_port(self,unallowed_ports,tests): + service_allowed=False + allowed = False + version = None + service = None + for port in unallowed_ports: + LOGGER.info('Checking unallowed port: ' + port['port']) + LOGGER.info('Looking for service: ' + port['service']) + LOGGER.debug('Unallowed Port Config: ' + str(port)) + if port['tcp_udp'] == 'tcp': + port_style = 'tcp_ports' + elif port['tcp_udp'] == 'udp': + port_style = 'udp_ports' + for test in tests: + LOGGER.debug('Checking test: ' + str(test)) + # Create a regular expression pattern to match the variable at the + # end of the string + port_service = r"\b" + re.escape(port['service']) + r"\b$" + service_match = re.search(port_service, test) + if service_match: + LOGGER.info("Service Matched: " + test) + service_config = tests[test] + service = port['service'] + for service_port in service_config[port_style]: + port_config = service_config[port_style][service_port] + service_allowed |= port_config['allowed'] + version = port_config['version'] if 'version' in port_config else None + if service_allowed: + LOGGER.info("Unallowed port detected for allowed service: " + service) + if version is not None: + allowed = self._check_version(service=service, + version_detected=self._scan_tcp_results[port['port']]['version'], + version_expected=version) + else: + allowed = True + if allowed: + LOGGER.info("Unallowed port exception for approved service: " + port['port']) + for u_port in self._unallowed_ports: + if port['port'] in u_port['port']: + self._unallowed_ports.remove(u_port) + break + break + + def _check_version(self,service,version_detected,version_expected): + """Check if the version specified for the service matches what was + detected by nmap. Since there is no consistency in how nmap service + results are returned, each service that needs a checked must be + implemented individually. If a service version is requested + that is not implemented, this test will provide a skip (None) + result. + """ + LOGGER.info("Checking version for service: " + service) + LOGGER.info("NMAP Version Detected: " + version_detected) + LOGGER.info("Version Expected: " + version_expected) + version_check = None + match service: + case "ssh": + version_check = f"protocol {version_expected}" in version_detected + case _: + LOGGER.info("No version check implemented for service: " + service + ". Skipping") + LOGGER.info("Version check result: " + str(version_check)) + return version_check + + def _scan_scripts(self, tests): + scan_results = {} + LOGGER.info("Checking for scan scripts") + for test in tests: + test_config = tests[test] + if "tcp_ports" in test_config: + for port in test_config["tcp_ports"]: + port_config = test_config["tcp_ports"][port] + if "service_scan" in port_config: + LOGGER.info("Service Scan Detected for: " + str(port)) + svc = port_config["service_scan"] + result = self._scan_tcp_with_script(svc["script"]) + scan_results.update(result) + if "udp_ports" in test_config: + for port in test_config["udp_ports"]: + if "service_scan" in port: + LOGGER.info("Service Scan Detected for: " + str(port)) + svc = port["service_scan"] + result = self._scan_udp_with_script(svc["script"], port) + scan_results.update(result) + self._script_scan_results = scan_results + + def _scan_tcp_with_script(self, script_name, ports=None): + LOGGER.info("Running TCP nmap scan with script " + script_name) + scan_options = " -v -n T3 --host-timeout=6m -A --script " + script_name + port_options = " --open " + if ports is None: + port_options += " -p- " + else: + port_options += " -p" + ports + " " + results_file = f"/runtime/output/{self._module_name}-script_name.log" + nmap_options = scan_options + port_options + " " + results_file + " -oX -" + nmap_results = util.run_command("nmap " + nmap_options + " " + + self._device_ipv4_addr)[0] + LOGGER.info("Nmap TCP script scan complete") + nmap_results_json = self._nmap_results_to_json(nmap_results) + return self._process_nmap_json_results(nmap_results_json=nmap_results_json) + + def _scan_udp_with_script(self, script_name, ports=None): + LOGGER.info("Running UDP nmap scan with script " + script_name) + scan_options = " --sU -Pn -n --script " + script_name + port_options = " --open " + if ports is None: + port_options += " -p- " + else: + port_options += " -p" + ports + " " + nmap_options = scan_options + port_options + " -oX - " + nmap_results = util.run_command("nmap " + nmap_options + + self._device_ipv4_addr)[0] + LOGGER.info("Nmap UDP script scan complete") + nmap_results_json = self._nmap_results_to_json(nmap_results) + return self._process_nmap_json_results(nmap_results_json=nmap_results_json) + + def _scan_tcp_ports(self, tests): + max_port = 65535 + LOGGER.info("Running nmap TCP port scan") + nmap_results = util.run_command( + f"""nmap --open -sT -sV -Pn -v -p 1-{max_port} + --version-intensity 7 -T4 -oX - {self._device_ipv4_addr}""")[0] + + LOGGER.info("TCP port scan complete") + nmap_results_json = self._nmap_results_to_json(nmap_results) + self._scan_tcp_results = self._process_nmap_json_results( + nmap_results_json=nmap_results_json) + + def _scan_udp_ports(self, tests): + ports = [] + for test in tests: + test_config = tests[test] + if "udp_ports" in test_config: + for port in test_config["udp_ports"]: + ports.append(port) + if len(ports) > 0: + port_list = ",".join(ports) + LOGGER.info("Running nmap UDP port scan") + LOGGER.info("UDP ports: " + str(port_list)) + nmap_results = util.run_command( + f"nmap -sU -sV -p {port_list} -oX - {self._device_ipv4_addr}")[0] + LOGGER.info("UDP port scan complete") + nmap_results_json = self._nmap_results_to_json(nmap_results) + self._scan_udp_results = self._process_nmap_json_results( + nmap_results_json=nmap_results_json) + + def _nmap_results_to_json(self,nmap_results): + try: + xml_data = xmltodict.parse(nmap_results) + json_data = json.dumps(xml_data, indent=4) + return json.loads(json_data) + + except Exception as e: + LOGGER.error(f"Error parsing Nmap output: {e}") + + def _process_nmap_json_results(self,nmap_results_json): + LOGGER.debug("nmap results\n" + json.dumps(nmap_results_json,indent=2)) + results = {} + if "ports" in nmap_results_json["nmaprun"]["host"]: + ports = nmap_results_json["nmaprun"]["host"]["ports"] + # Checking if an object is a JSON object + if isinstance(ports["port"], dict): + results.update(self._json_port_to_dict(ports["port"])) + elif isinstance(ports["port"], list): + for port in ports["port"]: + results.update(self._json_port_to_dict(port)) + return results + + def _json_port_to_dict(self,port_json): + port_result = {} + port = {} + port["tcp_udp"] = port_json["@protocol"] + port["state"] = port_json["state"]["@state"] + port["service"] = port_json["service"]["@name"] + port["version"] = "" + if "@version" in port_json["service"]: + port["version"] += port_json["service"]["@version"] + if "@extrainfo" in port_json["service"]: + port["version"] += " " + port_json["service"]["@extrainfo"] + port_result = {port_json["@portid"]:port} + return port_result \ No newline at end of file diff --git a/test_orc/modules/nmap/python/src/run.py b/modules/test/nmap/python/src/run.py similarity index 58% rename from test_orc/modules/nmap/python/src/run.py rename to modules/test/nmap/python/src/run.py index 959e30f87..5e33451d9 100644 --- a/test_orc/modules/nmap/python/src/run.py +++ b/modules/test/nmap/python/src/run.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Run NMAP test module""" import argparse import signal @@ -6,8 +20,8 @@ from nmap_module import NmapModule -LOGGER = logger.get_logger('test_module') - +LOG_NAME = "nmap_runner" +LOGGER = logger.get_logger(LOG_NAME) class NmapModuleRunner: """Run the NMAP module tests.""" @@ -18,12 +32,19 @@ def __init__(self, module): signal.signal(signal.SIGTERM, self._handler) signal.signal(signal.SIGABRT, self._handler) signal.signal(signal.SIGQUIT, self._handler) + self.add_logger(module) - LOGGER.info('Starting nmap Module') + LOGGER.info('Starting nmap module') self._test_module = NmapModule(module) self._test_module.run_tests() + LOGGER.info("nmap test module finished") + + def add_logger(self, module): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, module) + def _handler(self, signum): LOGGER.debug('SigtermEnum: ' + str(signal.SIGTERM)) LOGGER.debug('Exit signal received: ' + str(signum)) diff --git a/net_orc/.gitignore b/net_orc/.gitignore deleted file mode 100644 index 2d77147eb..000000000 --- a/net_orc/.gitignore +++ /dev/null @@ -1,133 +0,0 @@ -# Runtime folder -runtime/ -.vscode/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ diff --git a/net_orc/docker-compose.yml b/net_orc/docker-compose.yml deleted file mode 100644 index 8c50d766a..000000000 --- a/net_orc/docker-compose.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: "3.7" - -services: - - base: - build: - context: network/modules/base - dockerfile: base.Dockerfile - image: test-run/base - container_name: tr-ct-base - - ovs: - depends_on: - - base - build: - context: network/modules/ovs - dockerfile: ovs.Dockerfile - image: test-run/ovs - network_mode: host - container_name: tr-ct-ovs - stdin_open: true - privileged: true - volumes: - - $PWD/network/modules/ovs/python:/ovs/python - # Mount host open vswitch socket to allow container - # access to control open vswitch on the host - - /var/run/openvswitch/db.sock:/var/run/openvswitch/db.sock - # Mount host network namespace to allow container - # access to assign proper namespaces to containers - - /var/run/netns:/var/run/netns - - netorch: - depends_on: - - base - build: - context: . - dockerfile: orchestrator.Dockerfile - image: test-run/orchestrator - network_mode: host - privileged: true - volumes: - - $PWD/cmd:/orchestrator/cmd - - $PWD/network:/orchestrator/network - - $PWD/python:/orchestrator/python - # Mount host docker socket to allow container access - # control docker containers on the host - - /var/run/docker.sock:/var/run/docker.sock - # Mount host open vswitch socket to allow container - # access to control open vswitch on the host - - /var/run/openvswitch/db.sock:/var/run/openvswitch/db.sock - # Mount host network namespace to allow container - # access to assign proper namespaces to containers - - /var/run/netns:/var/run/netns - # Mount the host process information to allow container - # access to configure docker containers and namespaces properly - - /proc:/proc - container_name: network_orchestrator - stdin_open: true - working_dir: /orchestrator - #entrypoint: ["cmd/start"] - # Give more time for stopping so when we stop the container it has - # time to stop all network services gracefuly - stop_grace_period: 60s - entrypoint: ["python3","-u","python/src/run.py"] diff --git a/net_orc/network/modules/ntp/ntp.Dockerfile b/net_orc/network/modules/ntp/ntp.Dockerfile deleted file mode 100644 index 3474a504e..000000000 --- a/net_orc/network/modules/ntp/ntp.Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -# Image name: test-run/ntp -FROM test-run/base:latest - -# Copy over all configuration files -COPY network/modules/ntp/conf /testrun/conf - -# Copy over all binary files -COPY network/modules/ntp/bin /testrun/bin - -# Copy over all python files -COPY network/modules/ntp/python /testrun/python - -EXPOSE 123/udp diff --git a/net_orc/network/modules/ovs/bin/start_network_service b/net_orc/network/modules/ovs/bin/start_network_service deleted file mode 100644 index 7c38f484a..000000000 --- a/net_orc/network/modules/ovs/bin/start_network_service +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e - -if [[ "$EUID" -ne 0 ]]; then - echo "Must run as root." - exit 1 -fi - -asyncRun() { - "$@" & - pid="$!" - trap "echo 'Stopping PID $pid'; kill -SIGTERM $pid" SIGINT SIGTERM - - # A signal emitted while waiting will make the wait command return code > 128 - # Let's wrap it in a loop that doesn't end before the process is indeed stopped - while kill -0 $pid > /dev/null 2>&1; do - wait - done -} - -# -u flag allows python print statements -# to be logged by docker by running unbuffered -asyncRun exec python3 -u /ovs/python/src/run.py \ No newline at end of file diff --git a/net_orc/network/modules/ovs/conf/module_config.json b/net_orc/network/modules/ovs/conf/module_config.json deleted file mode 100644 index 8a440d0ae..000000000 --- a/net_orc/network/modules/ovs/conf/module_config.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "config": { - "meta": { - "name": "ovs", - "display_name": "OVS", - "description": "Setup and configure Open vSwitch" - }, - "network": { - "interface": "veth0", - "enable_wan": false, - "ip_index": 6, - "host": true - }, - "docker": { - "depends_on": "base", - "mounts": [ - { - "source": "runtime/network", - "target": "/runtime/network" - } - ] - } - } -} \ No newline at end of file diff --git a/net_orc/network/modules/ovs/ovs.Dockerfile b/net_orc/network/modules/ovs/ovs.Dockerfile deleted file mode 100644 index cd4710e66..000000000 --- a/net_orc/network/modules/ovs/ovs.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Image name: test-run/orchestrator -FROM test-run/base:latest - -#Update and get all additional requirements not contained in the base image -RUN apt-get update --fix-missing - -#Install openvswitch -RUN apt-get install -y openvswitch-switch - -# Copy over all configuration files -COPY network/modules/ovs/conf /testrun/conf - -# Copy over all binary files -COPY network/modules/ovs/bin /testrun/bin - -# Copy over all python files -COPY network/modules/ovs/python /testrun/python - -#Install all python requirements for the module -RUN pip3 install -r /testrun/python/requirements.txt \ No newline at end of file diff --git a/net_orc/network/modules/ovs/python/requirements.txt b/net_orc/network/modules/ovs/python/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/net_orc/network/modules/ovs/python/src/logger.py b/net_orc/network/modules/ovs/python/src/logger.py deleted file mode 100644 index 23e697e43..000000000 --- a/net_orc/network/modules/ovs/python/src/logger.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Sets up the logger to be used for the ovs modules.""" -import logging - -LOGGERS = {} -_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' -_DATE_FORMAT = '%b %02d %H:%M:%S' - -# Set level to debug if set as runtime flag -logging.basicConfig(format=_LOG_FORMAT, - datefmt=_DATE_FORMAT, - level=logging.INFO) - -def get_logger(name): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - return LOGGERS[name] diff --git a/net_orc/network/modules/ovs/python/src/ovs_control.py b/net_orc/network/modules/ovs/python/src/ovs_control.py deleted file mode 100644 index 765c50f92..000000000 --- a/net_orc/network/modules/ovs/python/src/ovs_control.py +++ /dev/null @@ -1,105 +0,0 @@ -"""OVS Control Module""" -import json -import logger -import util - -CONFIG_FILE = '/ovs/conf/system.json' -DEVICE_BRIDGE = 'tr-d' -INTERNET_BRIDGE = 'tr-c' -LOGGER = logger.get_logger('ovs_ctrl') - -class OVSControl: - """OVS Control""" - def __init__(self): - self._int_intf = None - self._dev_intf = None - self._load_config() - - def add_bridge(self, bridge_name): - LOGGER.info('Adding OVS Bridge: ' + bridge_name) - # Create the bridge using ovs-vsctl commands - # Uses the --may-exist option to prevent failures - # if this bridge already exists by this name it won't fail - # and will not modify the existing bridge - success=util.run_command('ovs-vsctl --may-exist add-br ' + bridge_name) - return success - - def add_port(self,port, bridge_name): - LOGGER.info('Adding Port ' + port + ' to OVS Bridge: ' + bridge_name) - # Add a port to the bridge using ovs-vsctl commands - # Uses the --may-exist option to prevent failures - # if this port already exists on the bridge and will not - # modify the existing bridge - success=util.run_command(f"""ovs-vsctl --may-exist - add-port {bridge_name} {port}""") - return success - - def create_net(self): - LOGGER.info('Creating baseline network') - - # Create data plane - self.add_bridge(DEVICE_BRIDGE) - - # Create control plane - self.add_bridge(INTERNET_BRIDGE) - - # Remove IP from internet adapter - self.set_interface_ip(self._int_intf,'0.0.0.0') - - # Add external interfaces to data and control plane - self.add_port(self._dev_intf,DEVICE_BRIDGE) - self.add_port(self._int_intf,INTERNET_BRIDGE) - - # # Set ports up - self.set_bridge_up(DEVICE_BRIDGE) - self.set_bridge_up(INTERNET_BRIDGE) - - def delete_bridge(self,bridge_name): - LOGGER.info('Deleting OVS Bridge: ' + bridge_name) - # Delete the bridge using ovs-vsctl commands - # Uses the --if-exists option to prevent failures - # if this bridge does not exists - success=util.run_command('ovs-vsctl --if-exists del-br ' + bridge_name) - return success - - def _load_config(self): - LOGGER.info('Loading Configuration: ' + CONFIG_FILE) - with open(CONFIG_FILE, 'r', encoding='utf-8') as conf_file: - config_json = json.load(conf_file) - self._int_intf = config_json['internet_intf'] - self._dev_intf = config_json['device_intf'] - LOGGER.info('Configuration Loaded') - LOGGER.info('Internet Interface: ' + self._int_intf) - LOGGER.info('Device Interface: ' + self._dev_intf) - - def restore_net(self): - LOGGER.info('Restoring Network...') - # Delete data plane - self.delete_bridge(DEVICE_BRIDGE) - - # Delete control plane - self.delete_bridge(INTERNET_BRIDGE) - - LOGGER.info('Network is restored') - - def show_config(self): - LOGGER.info('Show current config of OVS') - success=util.run_command('ovs-vsctl show') - return success - - def set_bridge_up(self,bridge_name): - LOGGER.info('Setting Bridge device to up state: ' + bridge_name) - success=util.run_command('ip link set dev ' + bridge_name + ' up') - return success - - def set_interface_ip(self,interface, ip_addr): - LOGGER.info('Setting interface ' + interface + ' to ' + ip_addr) - # Remove IP from internet adapter - util.run_command('ifconfig ' + interface + ' 0.0.0.0') - -if __name__ == '__main__': - ovs = OVSControl() - ovs.create_net() - ovs.show_config() - ovs.restore_net() - ovs.show_config() diff --git a/net_orc/network/modules/ovs/python/src/run.py b/net_orc/network/modules/ovs/python/src/run.py deleted file mode 100644 index 5787a74e6..000000000 --- a/net_orc/network/modules/ovs/python/src/run.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Run OVS module""" -import logger -import signal -import sys -import time - -from ovs_control import OVSControl - -LOGGER = logger.get_logger('ovs_control_run') - -class OVSControlRun: - """Run the OVS module.""" - def __init__(self): - - signal.signal(signal.SIGINT, self.handler) - signal.signal(signal.SIGTERM, self.handler) - signal.signal(signal.SIGABRT, self.handler) - signal.signal(signal.SIGQUIT, self.handler) - - LOGGER.info('Starting OVS Control') - - # Get all components ready - self._ovs_control = OVSControl() - - self._ovs_control.restore_net() - - self._ovs_control.create_net() - - self._ovs_control.show_config() - - # Get network ready (via Network orchestrator) - LOGGER.info('Network is ready. Waiting for device information...') - - #Loop forever until process is stopped - while True: - LOGGER.info('OVS Running') - time.sleep(1000) - - # TODO: This time should be configurable (How long to hold before exiting, - # this could be infinite too) - #time.sleep(300) - - # Tear down network - #self._ovs_control.shutdown() - - def handler(self, signum): - LOGGER.info('SigtermEnum: ' + str(signal.SIGTERM)) - LOGGER.info('Exit signal received: ' + str(signum)) - if (signum == 2 or signal == signal.SIGTERM): - LOGGER.info('Exit signal received. Restoring network...') - self._ovs_control.shutdown() - sys.exit(1) - -ovs = OVSControlRun() diff --git a/net_orc/network/modules/ovs/python/src/util.py b/net_orc/network/modules/ovs/python/src/util.py deleted file mode 100644 index a3ebbb10a..000000000 --- a/net_orc/network/modules/ovs/python/src/util.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Provides basic utilities for a ovs module.""" -import subprocess -import logger - -LOGGER = logger.get_logger('util') - -def run_command(cmd): - success = False - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if process.returncode != 0: - err_msg = f'{stderr.strip()}. Code: {process.returncode}' - LOGGER.error('Command Failed: ' + cmd) - LOGGER.error('Error: ' + err_msg) - else: - msg = stdout.strip().decode('utf-8') - succ_msg = f'{msg}. Code: {process.returncode}' - LOGGER.info('Command Success: ' + cmd) - LOGGER.info('Success: ' + succ_msg) - success = True - return success diff --git a/net_orc/network/modules/template/python/src/template_main.py b/net_orc/network/modules/template/python/src/template_main.py deleted file mode 100644 index df2452550..000000000 --- a/net_orc/network/modules/template/python/src/template_main.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Python code for the template module.""" - -if __name__ == "__main__": - print("Template main") diff --git a/net_orc/network/modules/template/template.Dockerfile b/net_orc/network/modules/template/template.Dockerfile deleted file mode 100644 index 45f9da6d9..000000000 --- a/net_orc/network/modules/template/template.Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Image name: test-run/template -FROM test-run/base:latest - -# Copy over all configuration files -COPY network/modules/template/conf /testrun/conf - -# Load device binary files -COPY network/modules/template/bin /testrun/bin - -# Copy over all python files -COPY network/modules/template/python /testrun/python \ No newline at end of file diff --git a/net_orc/orchestrator.Dockerfile b/net_orc/orchestrator.Dockerfile deleted file mode 100644 index f062a33d4..000000000 --- a/net_orc/orchestrator.Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Image name: test-run/orchestrator -FROM test-run/base:latest - -#Update and get all additional requirements not contained in the base image -RUN apt-get update - -RUN apt-get install -y python3-pip curl openvswitch-switch - -#Download and install docker client -ENV DOCKERVERSION=20.10.2 -RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKERVERSION}.tgz \ - && tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker \ - && rm docker-${DOCKERVERSION}.tgz - -#Create a directory to load all the app files into -RUN mkdir /python - -#Load the requirements file -COPY python/requirements.txt /python - -#Install all python requirements for the module -RUN pip3 install -r python/requirements.txt diff --git a/net_orc/python/requirements.txt b/net_orc/python/requirements.txt deleted file mode 100644 index 5d8f29214..000000000 --- a/net_orc/python/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -docker -ipaddress -netifaces -scapy \ No newline at end of file diff --git a/net_orc/python/src/network_device.py b/net_orc/python/src/network_device.py deleted file mode 100644 index 1b856da16..000000000 --- a/net_orc/python/src/network_device.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Track device object information.""" -from dataclasses import dataclass - - -@dataclass -class NetworkDevice: - """Represents a physical device and it's configuration.""" - - mac_addr: str - ip_addr: str = None diff --git a/net_orc/python/src/network_event.py b/net_orc/python/src/network_event.py deleted file mode 100644 index f56adf494..000000000 --- a/net_orc/python/src/network_event.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Specify the various types of network events to be reported.""" -from enum import Enum - - -class NetworkEvent(Enum): - """All possible network events.""" - DEVICE_DISCOVERED = 1 - DEVICE_STABLE = 2 - DHCP_LEASE_ACK = 3 diff --git a/resources/devices/Template/device_config.json b/resources/devices/template/device_config.json similarity index 100% rename from resources/devices/Template/device_config.json rename to resources/devices/template/device_config.json diff --git a/test_orc/modules/baseline/baseline.Dockerfile b/test_orc/modules/baseline/baseline.Dockerfile deleted file mode 100644 index 5b634e6ee..000000000 --- a/test_orc/modules/baseline/baseline.Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Image name: test-run/baseline-test -FROM test-run/base-test:latest - -# Copy over all configuration files -COPY modules/baseline/conf /testrun/conf - -# Load device binary files -COPY modules/baseline/bin /testrun/bin - -# Copy over all python files -COPY modules/baseline/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/dns/dns.Dockerfile b/test_orc/modules/dns/dns.Dockerfile deleted file mode 100644 index 7c3497bc3..000000000 --- a/test_orc/modules/dns/dns.Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Image name: test-run/baseline-test -FROM test-run/base-test:latest - -# Copy over all configuration files -COPY modules/dns/conf /testrun/conf - -# Load device binary files -COPY modules/dns/bin /testrun/bin - -# Copy over all python files -COPY modules/dns/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/nmap/nmap.Dockerfile b/test_orc/modules/nmap/nmap.Dockerfile deleted file mode 100644 index 12f23dde7..000000000 --- a/test_orc/modules/nmap/nmap.Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Image name: test-run/baseline-test -FROM test-run/base-test:latest - -# Copy over all configuration files -COPY modules/nmap/conf /testrun/conf - -# Load device binary files -COPY modules/nmap/bin /testrun/bin - -# Copy over all python files -COPY modules/nmap/python /testrun/python \ No newline at end of file diff --git a/test_orc/modules/nmap/python/src/nmap_module.py b/test_orc/modules/nmap/python/src/nmap_module.py deleted file mode 100644 index 876343a0f..000000000 --- a/test_orc/modules/nmap/python/src/nmap_module.py +++ /dev/null @@ -1,227 +0,0 @@ -"""NMAP test module""" -import time -import util -import json -import threading -from test_module import TestModule - -LOG_NAME = "test_nmap" -LOGGER = None - - -class NmapModule(TestModule): - """NMAP Test module""" - def __init__(self, module): - super().__init__(module_name=module, log_name=LOG_NAME) - self._unallowed_ports = [] - self._scan_tcp_results = None - self._udp_tcp_results = None - self._script_scan_results = None - global LOGGER - LOGGER = self._get_logger() - - def _security_nmap_ports(self, config): - LOGGER.info("Running security.nmap.ports test") - - # Delete the enabled key from the config if it exists - # to prevent it being treated as a test key - if "enabled" in config: - del config["enabled"] - - if self._device_ipv4_addr is not None: - # Run the monitor method asynchronously to keep this method non-blocking - self._tcp_scan_thread = threading.Thread(target=self._scan_tcp_ports, - args=(config, )) - self._udp_scan_thread = threading.Thread(target=self._scan_udp_ports, - args=(config, )) - self._script_scan_thread = threading.Thread(target=self._scan_scripts, - args=(config, )) - - self._tcp_scan_thread.daemon = True - self._udp_scan_thread.daemon = True - self._script_scan_thread.daemon = True - - self._tcp_scan_thread.start() - self._udp_scan_thread.start() - self._script_scan_thread.start() - - while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive( - ) or self._script_scan_thread.is_alive(): - time.sleep(1) - - LOGGER.debug("TCP scan results: " + str(self._scan_tcp_results)) - LOGGER.debug("UDP scan results: " + str(self._scan_udp_results)) - LOGGER.debug("Service scan results: " + str(self._script_scan_results)) - self._process_port_results(tests=config) - LOGGER.info("Unallowed Ports: " + str(self._unallowed_ports)) - LOGGER.info("Script scan results:\n" + - json.dumps(self._script_scan_results)) - return len(self._unallowed_ports) == 0 - else: - LOGGER.info("Device ip address not resolved, skipping") - return None - - def _process_port_results(self, tests): - for test in tests: - LOGGER.info("Checking results for test: " + str(test)) - self._check_scan_results(test_config=tests[test]) - - def _check_scan_results(self, test_config): - port_config = {} - if "tcp_ports" in test_config: - port_config.update(test_config["tcp_ports"]) - elif "udp_ports" in test_config: - port_config.update(test_config["udp_ports"]) - - scan_results = {} - if self._scan_tcp_results is not None: - scan_results.update(self._scan_tcp_results) - if self._scan_udp_results is not None: - scan_results.update(self._scan_udp_results) - if self._script_scan_results is not None: - scan_results.update(self._script_scan_results) - if port_config is not None: - for port, config in port_config.items(): - result = None - LOGGER.info("Checking port: " + str(port)) - LOGGER.debug("Port config: " + str(config)) - if port in scan_results: - if scan_results[port]["state"] == "open": - if not config["allowed"]: - LOGGER.info("Unallowed port open") - self._unallowed_ports.append(str(port)) - result = False - else: - LOGGER.info("Allowed port open") - result = True - else: - LOGGER.info("Port is closed") - result = True - else: - LOGGER.info("Port not detected, closed") - result = True - - if result is not None: - config["result"] = "compliant" if result else "non-compliant" - else: - config["result"] = "skipped" - - def _scan_scripts(self, tests): - scan_results = {} - LOGGER.info("Checing for scan scripts") - for test in tests: - test_config = tests[test] - if "tcp_ports" in test_config: - for port in test_config["tcp_ports"]: - port_config = test_config["tcp_ports"][port] - if "service_scan" in port_config: - LOGGER.info("Service Scan Detected for: " + str(port)) - svc = port_config["service_scan"] - scan_results.update(self._scan_tcp_with_script(svc["script"])) - if "udp_ports" in test_config: - for port in test_config["udp_ports"]: - if "service_scan" in port: - LOGGER.info("Service Scan Detected for: " + str(port)) - svc = port["service_scan"] - self._scan_udp_with_script(svc["script"], port) - scan_results.update(self._scan_tcp_with_script(svc["script"])) - self._script_scan_results = scan_results - - def _scan_tcp_with_script(self, script_name, ports=None): - LOGGER.info("Running TCP nmap scan with script " + script_name) - scan_options = " -v -n T3 --host-timeout=6m -A --script " + script_name - port_options = " --open " - if ports is None: - port_options += " -p- " - else: - port_options += " -p" + ports + " " - results_file = f"/runtime/output/{self._module_name}-script_name.log" - nmap_options = scan_options + port_options + " -oG " + results_file - nmap_results = util.run_command("nmap " + nmap_options + " " + - self._device_ipv4_addr)[0] - LOGGER.info("Nmap TCP script scan complete") - LOGGER.info("nmap script results\n" + str(nmap_results)) - return self._process_nmap_results(nmap_results=nmap_results) - - def _scan_udp_with_script(self, script_name, ports=None): - LOGGER.info("Running UDP nmap scan with script " + script_name) - scan_options = " --sU -Pn -n --script " + script_name - port_options = " --open " - if ports is None: - port_options += " -p- " - else: - port_options += " -p" + ports + " " - nmap_options = scan_options + port_options - nmap_results = util.run_command("nmap " + nmap_options + - self._device_ipv4_addr)[0] - LOGGER.info("Nmap UDP script scan complete") - LOGGER.info("nmap script results\n" + str(nmap_results)) - return self._process_nmap_results(nmap_results=nmap_results) - - def _scan_tcp_ports(self, tests): - max_port = 1000 - ports = [] - for test in tests: - test_config = tests[test] - if "tcp_ports" in test_config: - for port in test_config["tcp_ports"]: - if int(port) > max_port: - ports.append(port) - ports_to_scan = "1-" + str(max_port) - if len(ports) > 0: - ports_to_scan += "," + ",".join(ports) - LOGGER.info("Running nmap TCP port scan") - LOGGER.info("TCP ports: " + str(ports_to_scan)) - nmap_results = util.run_command(f"""nmap -sT -sV -Pn -v -p {ports_to_scan} - --version-intensity 7 -T4 {self._device_ipv4_addr}""")[0] - LOGGER.info("TCP port scan complete") - self._scan_tcp_results = self._process_nmap_results( - nmap_results=nmap_results) - - def _scan_udp_ports(self, tests): - ports = [] - for test in tests: - test_config = tests[test] - if "udp_ports" in test_config: - for port in test_config["udp_ports"]: - ports.append(port) - if len(ports) > 0: - port_list = ",".join(ports) - LOGGER.info("Running nmap UDP port scan") - LOGGER.info("UDP ports: " + str(port_list)) - nmap_results = util.run_command( - f"nmap -sU -sV -p {port_list} {self._device_ipv4_addr}")[0] - LOGGER.info("UDP port scan complete") - self._scan_udp_results = self._process_nmap_results( - nmap_results=nmap_results) - - def _process_nmap_results(self, nmap_results): - results = {} - LOGGER.info("nmap results\n" + str(nmap_results)) - if nmap_results: - if "Service Info" in nmap_results: - rows = nmap_results.split("PORT")[1].split("Service Info")[0].split( - "\n") - elif "PORT" in nmap_results: - rows = nmap_results.split("PORT")[1].split("MAC Address")[0].split("\n") - if rows: - for result in rows[1:-1]: # Iterate skipping the header and tail rows - cols = result.split() - port = cols[0].split("/")[0] - # If results do not start with a a port number, - # it is likely a bleed over from previous result so - # we need to ignore it - if port.isdigit(): - version = "" - if len(cols) > 3: - # recombine full version information that may contain spaces - version = " ".join(cols[3:]) - port_result = { - cols[0].split("/")[0]: { - "state": cols[1], - "service": cols[2], - "version": version - } - } - results.update(port_result) - return results diff --git a/test_orc/python/requirements.txt b/test_orc/python/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/testing/test_baseline b/testing/test_baseline index d7fc1e5c5..36d21fa5e 100755 --- a/testing/test_baseline +++ b/testing/test_baseline @@ -3,6 +3,8 @@ TESTRUN_OUT=/tmp/testrun.log +ifconfig + # Setup requirements sudo apt-get update sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils @@ -18,13 +20,10 @@ sudo docker network create -d macvlan -o parent=endev0b endev0 # Start OVS sudo /usr/share/openvswitch/scripts/ovs-ctl start -# Fix due to ordering -sudo docker build ./net_orc/ -t test-run/base -f net_orc/network/modules/base/base.Dockerfile - # Build Test Container sudo docker build ./testing/docker/ci_baseline -t ci1 -f ./testing/docker/ci_baseline/Dockerfile -cat <conf/system.json +cat <local/system.json { "network": { "device_intf": "endev0a", diff --git a/testing/test_baseline.py b/testing/test_baseline.py index b356983dd..246857581 100644 --- a/testing/test_baseline.py +++ b/testing/test_baseline.py @@ -1,3 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import pytest import re From 54b4088edb9a725ce69ec42897716af2006b2b62 Mon Sep 17 00:00:00 2001 From: J Boddey Date: Wed, 28 Jun 2023 00:12:41 +0100 Subject: [PATCH 2/2] Sprint 8 Hotfix (#54) * Fix connection results.json * Re add try/catch * Fix log level * Debug test module load order * Add depends on to nmap module * Remove logging change --- cmd/start | 21 ----- framework/python/src/common/logger.py | 2 +- framework/python/src/common/util.py | 44 ++++++++++- framework/python/src/core/device.py | 2 +- framework/python/src/core/testrun.py | 37 ++++++--- .../src/net_orc/network_orchestrator.py | 76 +++---------------- .../python/src/test_orc/test_orchestrator.py | 64 ++++------------ modules/test/base/python/src/test_module.py | 1 - modules/test/nmap/conf/module_config.json | 1 + resources/devices/template/device_config.json | 11 ++- 10 files changed, 101 insertions(+), 158 deletions(-) diff --git a/cmd/start b/cmd/start index 55d2e52eb..17bc2af6c 100755 --- a/cmd/start +++ b/cmd/start @@ -22,25 +22,4 @@ source venv/bin/activate export PYTHONPATH="$PWD/framework/python/src" python -u framework/python/src/core/test_runner.py $@ -# TODO: Work in progress code for containerization of OVS module -# asyncRun() { -# "$@" & -# pid="$!" -# echo "PID Running: " $pid -# trap "echo 'Stopping PID $pid'; kill -SIGTERM $pid" SIGINT SIGTERM - -# sleep 10 - -# # A signal emitted while waiting will make the wait command return code > 128 -# # Let's wrap it in a loop that doesn't end before the process is indeed stopped -# while kill -0 $pid > /dev/null 2>&1; do -# #while $(kill -0 $pid 2>/dev/null); do -# wait -# done -# } - -# # -u flag allows python print statements -# # to be logged by docker by running unbuffered -# asyncRun python3 -u python/src/run.py $@ - deactivate \ No newline at end of file diff --git a/framework/python/src/common/logger.py b/framework/python/src/common/logger.py index 539767f53..8dd900fea 100644 --- a/framework/python/src/common/logger.py +++ b/framework/python/src/common/logger.py @@ -21,7 +21,7 @@ _LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' _DATE_FORMAT = '%b %02d %H:%M:%S' _DEFAULT_LEVEL = logging.INFO -_CONF_DIR = 'conf' +_CONF_DIR = 'local' _CONF_FILE_NAME = 'system.json' # Set log level diff --git a/framework/python/src/common/util.py b/framework/python/src/common/util.py index 1ffe70651..441b93224 100644 --- a/framework/python/src/common/util.py +++ b/framework/python/src/common/util.py @@ -13,6 +13,8 @@ # limitations under the License. """Provides basic utilities for the network orchestrator.""" +import getpass +import os import subprocess import shlex from common import logger @@ -37,7 +39,7 @@ def run_command(cmd, output=True): if process.returncode != 0 and output: err_msg = f'{stderr.strip()}. Code: {process.returncode}' - LOGGER.error('Command Failed: ' + cmd) + LOGGER.error('Command failed: ' + cmd) LOGGER.error('Error: ' + err_msg) else: success = True @@ -50,6 +52,44 @@ def run_command(cmd, output=True): def interface_exists(interface): return interface in netifaces.interfaces() - def prettify(mac_string): return ':'.join([f'{ord(b):02x}' for b in mac_string]) + +def get_host_user(): + user = get_os_user() + + # If primary method failed, try secondary + if user is None: + user = get_user() + + return user + +def get_os_user(): + user = None + try: + user = os.getlogin() + except OSError: + # Handle the OSError exception + LOGGER.error('An OS error occured whilst calling os.getlogin()') + except Exception: + # Catch any other unexpected exceptions + LOGGER.error('An unknown exception occured whilst calling os.getlogin()') + return user + +def get_user(): + user = None + try: + user = getpass.getuser() + except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: + # Handle specific exceptions individually + if isinstance(e, KeyError): + LOGGER.error('USER environment variable not set or unavailable.') + elif isinstance(e, ImportError): + LOGGER.error('Unable to import the getpass module.') + elif isinstance(e, ModuleNotFoundError): + LOGGER.error('The getpass module was not found.') + elif isinstance(e, OSError): + LOGGER.error('An OS error occurred while retrieving the username.') + else: + LOGGER.error('An exception occurred:', e) + return user diff --git a/framework/python/src/core/device.py b/framework/python/src/core/device.py index 44f275bdf..efce2dba1 100644 --- a/framework/python/src/core/device.py +++ b/framework/python/src/core/device.py @@ -22,6 +22,6 @@ class Device(NetworkDevice): """Represents a physical device and it's configuration.""" - make: str = None + manufacturer: str = None model: str = None test_modules: str = None diff --git a/framework/python/src/core/testrun.py b/framework/python/src/core/testrun.py index d613410e9..a91736e95 100644 --- a/framework/python/src/core/testrun.py +++ b/framework/python/src/core/testrun.py @@ -25,7 +25,7 @@ import json import signal import time -from common import logger +from common import logger, util # Locate parent directory current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -46,7 +46,7 @@ LOCAL_DEVICES_DIR = 'local/devices' RESOURCE_DEVICES_DIR = 'resources/devices' DEVICE_CONFIG = 'device_config.json' -DEVICE_MAKE = 'make' +DEVICE_MANUFACTURER = 'manufacturer' DEVICE_MODEL = 'model' DEVICE_MAC_ADDR = 'mac_addr' DEVICE_TEST_MODULES = 'test_modules' @@ -76,7 +76,6 @@ def __init__(self, self._net_orc = net_orc.NetworkOrchestrator( config_file=config_file_abs, validate=validate, - async_monitor=not self._net_only, single_intf = self._single_intf) self._test_orc = test_orc.TestOrchestrator(self._net_orc) @@ -85,17 +84,30 @@ def start(self): self._load_all_devices() + self._start_network() + if self._net_only: LOGGER.info('Network only option configured, no tests will be run') - self._start_network() + + self._net_orc.listener.register_callback( + self._device_discovered, + [NetworkEvent.DEVICE_DISCOVERED] + ) + + self._net_orc.start_listener() + LOGGER.info('Waiting for devices on the network...') + + while True: + time.sleep(RUNTIME) + else: - self._start_network() self._test_orc.start() self._net_orc.listener.register_callback( self._device_stable, [NetworkEvent.DEVICE_STABLE] ) + self._net_orc.listener.register_callback( self._device_discovered, [NetworkEvent.DEVICE_DISCOVERED] @@ -106,13 +118,13 @@ def start(self): time.sleep(RUNTIME) - if not self._test_orc.test_in_progress(): - LOGGER.info('Timed out whilst waiting for device') + if not (self._test_orc.test_in_progress() or self._net_orc.monitor_in_progress()): + LOGGER.info('Timed out whilst waiting for device or stopping due to test completion') else: - while self._test_orc.test_in_progress(): + while self._test_orc.test_in_progress() or self._net_orc.monitor_in_progress(): time.sleep(5) - self.stop() + self.stop() def stop(self, kill=False): self._stop_tests() @@ -157,18 +169,19 @@ def _load_devices(self, device_dir): LOGGER.debug('Loading devices from ' + device_dir) os.makedirs(device_dir, exist_ok=True) + util.run_command(f'chown -R {util.get_host_user()} {device_dir}') for device_folder in os.listdir(device_dir): with open(os.path.join(device_dir, device_folder, DEVICE_CONFIG), encoding='utf-8') as device_config_file: device_config_json = json.load(device_config_file) - device_make = device_config_json.get(DEVICE_MAKE) + device_manufacturer = device_config_json.get(DEVICE_MANUFACTURER) device_model = device_config_json.get(DEVICE_MODEL) mac_addr = device_config_json.get(DEVICE_MAC_ADDR) test_modules = device_config_json.get(DEVICE_TEST_MODULES) - device = Device(make=device_make, + device = Device(manufacturer=device_manufacturer, model=device_model, mac_addr=mac_addr, test_modules=json.dumps(test_modules)) @@ -184,7 +197,7 @@ def _device_discovered(self, mac_addr): device = self.get_device(mac_addr) if device is not None: LOGGER.info( - f'Discovered {device.make} {device.model} on the network') + f'Discovered {device.manufacturer} {device.model} on the network') else: device = Device(mac_addr=mac_addr) self._devices.append(device) diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index 643dc4def..499ce954b 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -21,8 +21,6 @@ import shutil import subprocess import sys -import time -import threading import docker from docker.types import Mount from common import logger @@ -41,7 +39,6 @@ TEST_DIR = 'test' MONITOR_PCAP = 'monitor.pcap' NET_DIR = 'runtime/network' -#NETWORK_MODULES_DIR = 'network/modules' NETWORK_MODULES_DIR = 'modules/network' NETWORK_MODULE_METADATA = 'conf/module_config.json' DEVICE_BRIDGE = 'tr-d' @@ -56,21 +53,18 @@ DEFAULT_RUNTIME = 1200 DEFAULT_MONITOR_PERIOD = 300 -RUNTIME = 1500 - - class NetworkOrchestrator: """Manage and controls a virtual testing network.""" def __init__(self, config_file=CONFIG_FILE, validate=True, - async_monitor=False, single_intf=False): self._runtime = DEFAULT_RUNTIME self._startup_timeout = DEFAULT_STARTUP_TIMEOUT self._monitor_period = DEFAULT_MONITOR_PERIOD + self._monitor_in_progress = False self._int_intf = None self._dev_intf = None @@ -80,7 +74,6 @@ def __init__(self, self._net_modules = [] self._devices = [] self.validate = validate - self.async_monitor = async_monitor self._path = os.path.dirname( os.path.dirname( @@ -99,7 +92,7 @@ def start(self): LOGGER.debug('Starting network orchestrator') - self._host_user = self._get_host_user() + self._host_user = util.get_host_user() # Get all components ready self.load_network_modules() @@ -109,14 +102,6 @@ def start(self): self.start_network() - if self.async_monitor: - # Run the monitor method asynchronously to keep this method non-blocking - self._monitor_thread = threading.Thread(target=self.monitor_network) - self._monitor_thread.daemon = True - self._monitor_thread.start() - else: - self.monitor_network() - def start_network(self): """Start the virtual testing network.""" LOGGER.info('Starting network') @@ -130,7 +115,7 @@ def start_network(self): self.validator.start() # Get network ready (via Network orchestrator) - LOGGER.info('Network is ready.') + LOGGER.debug('Network is ready') def start_listener(self): self.listener.start_listener() @@ -151,13 +136,6 @@ def stop_network(self, kill=False): self.stop_networking_services(kill=kill) self.restore_net() - def monitor_network(self): - # TODO: This time should be configurable (How long to hold before exiting, - # this could be infinite too) - time.sleep(RUNTIME) - - self.stop() - def load_config(self, config_file=None): if config_file is None: # If not defined, use relative pathing to local file @@ -178,8 +156,11 @@ def load_config(self, config_file=None): def _device_discovered(self, mac_addr): + self._monitor_in_progress = True + LOGGER.debug( f'Discovered device {mac_addr}. Waiting for device to obtain IP') + device = self._get_device(mac_addr=mac_addr) device_runtime_dir = os.path.join(RUNTIME_DIR, TEST_DIR, @@ -204,6 +185,9 @@ def _device_discovered(self, mac_addr): self._start_device_monitor(device) + def monitor_in_progress(self): + return self._monitor_in_progress + def _device_has_ip(self, packet): device = self._get_device(mac_addr=packet.src) if device is None or device.ip_addr is None: @@ -225,6 +209,8 @@ def _start_device_monitor(self, device): wrpcap( os.path.join(RUNTIME_DIR, TEST_DIR, device.mac_addr.replace(':', ''), 'monitor.pcap'), packet_capture) + + self._monitor_in_progress = False self.listener.call_callback(NetworkEvent.DEVICE_STABLE, device.mac_addr) def _get_device(self, mac_addr): @@ -490,46 +476,6 @@ def _start_network_service(self, net_module): if network != 'host': self._attach_service_to_network(net_module) - def _get_host_user(self): - user = self._get_os_user() - - # If primary method failed, try secondary - if user is None: - user = self._get_user() - - LOGGER.debug("Network orchestrator host user: " + user) - return user - - def _get_os_user(self): - user = None - try: - user = os.getlogin() - except OSError as e: - # Handle the OSError exception - LOGGER.error("An OS error occurred while retrieving the login name.") - except Exception as e: - # Catch any other unexpected exceptions - LOGGER.error("An exception occurred:", e) - return user - - def _get_user(self): - user = None - try: - user = getpass.getuser() - except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: - # Handle specific exceptions individually - if isinstance(e, KeyError): - LOGGER.error("USER environment variable not set or unavailable.") - elif isinstance(e, ImportError): - LOGGER.error("Unable to import the getpass module.") - elif isinstance(e, ModuleNotFoundError): - LOGGER.error("The getpass module was not found.") - elif isinstance(e, OSError): - LOGGER.error("An OS error occurred while retrieving the username.") - else: - LOGGER.error("An exception occurred:", e) - return user - def _stop_service_module(self, net_module, kill=False): LOGGER.debug('Stopping Service container ' + net_module.container_name) try: diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index 58c1944f8..4d39deab8 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -59,7 +59,7 @@ def start(self): LOGGER.debug("Starting test orchestrator") # Setup the output directory - self._host_user = self._get_host_user() + self._host_user = util.get_host_user() os.makedirs(RUNTIME_DIR, exist_ok=True) util.run_command(f'chown -R {self._host_user} {RUNTIME_DIR}') @@ -78,19 +78,19 @@ def run_test_modules(self, device): for module in self._test_modules: self._run_test_module(module, device) LOGGER.info("All tests complete") - LOGGER.info( - f"""Completed running test \ -modules on device with mac \ -addr {device.mac_addr}""") + + # Wait for all containers to stop (this should actually check in future) + time.sleep(5) + self._generate_results(device) self._test_in_progress = False def _generate_results(self, device): results = {} results["device"] = {} - if device.make is not None: - results["device"]["make"] = device.make - if device.make is not None: + if device.manufacturer is not None: + results["device"]["manufacturer"] = device.manufacturer + if device.manufacturer is not None: results["device"]["model"] = device.model results["device"]["mac_addr"] = device.mac_addr for module in self._test_modules: @@ -100,12 +100,12 @@ def _generate_results(self, device): device.mac_addr.replace(":", "") + "/" + module.name) results_file = f"{container_runtime_dir}/{module.name}-result.json" try: - with open(results_file, "r", encoding="UTF-8") as f: + with open(results_file, "r", encoding="utf-8-sig") as f: module_results = json.load(f) results[module.name] = module_results except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: - LOGGER.error("Error occured whilst running module " + module.name) + LOGGER.error("Error occured whilst obbtaining results for module " + module.name) LOGGER.debug(results_error) out_file = os.path.join( @@ -237,47 +237,6 @@ def _get_module_container(self, module): LOGGER.error(error) return container - def _get_host_user(self): - user = self._get_os_user() - - # If primary method failed, try secondary - if user is None: - user = self._get_user() - - LOGGER.debug("Test orchestrator host user: " + user) - return user - - def _get_os_user(self): - user = None - try: - user = os.getlogin() - except OSError as e: - # Handle the OSError exception - LOGGER.error("An OS error occurred while retrieving the login name.") - except Exception as e: - # Catch any other unexpected exceptions - LOGGER.error("An exception occurred:", e) - return user - - def _get_user(self): - user = None - try: - user = getpass.getuser() - except (KeyError, ImportError, ModuleNotFoundError, OSError) as e: - # Handle specific exceptions individually - if isinstance(e, KeyError): - LOGGER.error("USER environment variable not set or unavailable.") - elif isinstance(e, ImportError): - LOGGER.error("Unable to import the getpass module.") - elif isinstance(e, ModuleNotFoundError): - LOGGER.error("The getpass module was not found.") - elif isinstance(e, OSError): - LOGGER.error("An OS error occurred while retrieving the username.") - else: - LOGGER.error("An exception occurred:", e) - return user - - def _load_test_modules(self): """Load network modules from module_config.json.""" LOGGER.debug("Loading test modules from /" + TEST_MODULES_DIR) @@ -296,6 +255,8 @@ def _load_test_modules(self): def _load_test_module(self, module_dir): """Import module configuration from module_config.json.""" + LOGGER.debug("Loading test module " + module_dir) + modules_dir = os.path.join(self._path, TEST_MODULES_DIR) # Load basic module information @@ -337,6 +298,7 @@ def build_test_modules(self): def _build_test_module(self, module): LOGGER.debug("Building docker image for module " + module.dir_name) + client = docker.from_env() try: client.images.build( diff --git a/modules/test/base/python/src/test_module.py b/modules/test/base/python/src/test_module.py index f29668bb2..5342e36f8 100644 --- a/modules/test/base/python/src/test_module.py +++ b/modules/test/base/python/src/test_module.py @@ -65,7 +65,6 @@ def _get_device_tests(self, device_test_module): return module_tests def _get_device_test_module(self): - # TODO: Make DEVICE_TEST_MODULES a static string if 'DEVICE_TEST_MODULES' in os.environ: test_modules = json.loads(os.environ['DEVICE_TEST_MODULES']) if self._module_name in test_modules: diff --git a/modules/test/nmap/conf/module_config.json b/modules/test/nmap/conf/module_config.json index aafde4c03..292eced8b 100644 --- a/modules/test/nmap/conf/module_config.json +++ b/modules/test/nmap/conf/module_config.json @@ -7,6 +7,7 @@ }, "network": true, "docker": { + "depends_on": "base", "enable_container": true, "timeout": 600 }, diff --git a/resources/devices/template/device_config.json b/resources/devices/template/device_config.json index 7a3d4441c..3bb804b22 100644 --- a/resources/devices/template/device_config.json +++ b/resources/devices/template/device_config.json @@ -1,5 +1,5 @@ { - "make": "Manufacturer X", + "manufacturer": "Manufacturer X", "model": "Device X", "mac_addr": "aa:bb:cc:dd:ee:ff", "test_modules": { @@ -15,9 +15,9 @@ } }, "baseline": { - "enabled": true, + "enabled": false, "tests": { - "baseline.passe": { + "baseline.non-compliant": { "enabled": true }, "baseline.pass": { @@ -74,6 +74,9 @@ "tcp_ports": { "80": { "allowed": false + }, + "443": { + "allowed": true } } }, @@ -144,4 +147,4 @@ } } } -} \ No newline at end of file +}