From 227694d600072634019ccf656326cde981d8bc2e Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 25 May 2023 11:30:38 +0100 Subject: [PATCH 01/12] More formatting fixes --- .gitignore | 135 +---- net_orc/network/modules/ntp/ntp-server.py | 532 +++++++++--------- net_orc/network/modules/radius/conf/ca.crt | 54 +- net_orc/python/src/network_orchestrator.py | 2 +- test_orc/modules/dns/python/src/dns_module.py | 132 ++--- test_orc/modules/dns/python/src/run.py | 61 +- .../modules/nmap/python/src/nmap_module.py | 426 +++++++------- test_orc/python/src/test_orchestrator.py | 62 +- testing/test_baseline.py | 41 +- 9 files changed, 664 insertions(+), 781 deletions(-) diff --git a/.gitignore b/.gitignore index db1580ffb..5dfc1f6f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,136 +1,7 @@ -# Runtime folder runtime/ venv/ .vscode/ - +error +pylint.out local/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +__pycache__/ \ No newline at end of file diff --git a/net_orc/network/modules/ntp/ntp-server.py b/net_orc/network/modules/ntp/ntp-server.py index ace3099b0..9d6a6da8e 100644 --- a/net_orc/network/modules/ntp/ntp-server.py +++ b/net_orc/network/modules/ntp/ntp-server.py @@ -8,308 +8,300 @@ import select taskQueue = queue.Queue() -stopFlag = False +stop_flag = False def system_to_ntp_time(timestamp): - """Convert a system time to a NTP time. + """Convert a system time to a NTP time. - Parameters: - timestamp -- timestamp in system time + Parameters: + timestamp -- timestamp in system time - Returns: - corresponding NTP time - """ - return timestamp + NTP.NTP_DELTA + Returns: + corresponding NTP time + """ + return timestamp + NTP.NTP_DELTA def _to_int(timestamp): - """Return the integral part of a timestamp. + """Return the integral part of a timestamp. - Parameters: - timestamp -- NTP timestamp + Parameters: + timestamp -- NTP timestamp - Retuns: - integral part - """ - return int(timestamp) + Retuns: + integral part + """ + return int(timestamp) def _to_frac(timestamp, n=32): - """Return the fractional part of a timestamp. + """Return the fractional part of a timestamp. - Parameters: - timestamp -- NTP timestamp - n -- number of bits of the fractional part + Parameters: + timestamp -- NTP timestamp + n -- number of bits of the fractional part - Retuns: - fractional part - """ - return int(abs(timestamp - _to_int(timestamp)) * 2**n) + Retuns: + fractional part + """ + return int(abs(timestamp - _to_int(timestamp)) * 2**n) def _to_time(integ, frac, n=32): - """Return a timestamp from an integral and fractional part. + """Return a timestamp from an integral and fractional part. - Parameters: - integ -- integral part - frac -- fractional part - n -- number of bits of the fractional part - - Retuns: - timestamp - """ - return integ + float(frac)/2**n - + Parameters: + integ -- integral part + frac -- fractional part + n -- number of bits of the fractional part + Retuns: + timestamp + """ + return integ + float(frac)/2**n class NTPException(Exception): - """Exception raised by this module.""" - pass - + """Exception raised by this module.""" + pass class NTP: - """Helper class defining constants.""" - - _SYSTEM_EPOCH = datetime.date(*time.gmtime(0)[0:3]) - """system epoch""" - _NTP_EPOCH = datetime.date(1900, 1, 1) - """NTP epoch""" - NTP_DELTA = (_SYSTEM_EPOCH - _NTP_EPOCH).days * 24 * 3600 - """delta between system and NTP time""" - - REF_ID_TABLE = { - 'DNC': "DNC routing protocol", - 'NIST': "NIST public modem", - 'TSP': "TSP time protocol", - 'DTS': "Digital Time Service", - 'ATOM': "Atomic clock (calibrated)", - 'VLF': "VLF radio (OMEGA, etc)", - 'callsign': "Generic radio", - 'LORC': "LORAN-C radionavidation", - 'GOES': "GOES UHF environment satellite", - 'GPS': "GPS UHF satellite positioning", - } - """reference identifier table""" - - STRATUM_TABLE = { - 0: "unspecified", - 1: "primary reference", - } - """stratum table""" - - MODE_TABLE = { - 0: "unspecified", - 1: "symmetric active", - 2: "symmetric passive", - 3: "client", - 4: "server", - 5: "broadcast", - 6: "reserved for NTP control messages", - 7: "reserved for private use", - } - """mode table""" - - LEAP_TABLE = { - 0: "no warning", - 1: "last minute has 61 seconds", - 2: "last minute has 59 seconds", - 3: "alarm condition (clock not synchronized)", - } - """leap indicator table""" + """Helper class defining constants.""" + + _SYSTEM_EPOCH = datetime.date(*time.gmtime(0)[0:3]) + """system epoch""" + _NTP_EPOCH = datetime.date(1900, 1, 1) + """NTP epoch""" + NTP_DELTA = (_SYSTEM_EPOCH - _NTP_EPOCH).days * 24 * 3600 + """delta between system and NTP time""" + + REF_ID_TABLE = { + 'DNC': "DNC routing protocol", + 'NIST': "NIST public modem", + 'TSP': "TSP time protocol", + 'DTS': "Digital Time Service", + 'ATOM': "Atomic clock (calibrated)", + 'VLF': "VLF radio (OMEGA, etc)", + 'callsign': "Generic radio", + 'LORC': "LORAN-C radionavidation", + 'GOES': "GOES UHF environment satellite", + 'GPS': "GPS UHF satellite positioning", + } + """reference identifier table""" + + STRATUM_TABLE = { + 0: "unspecified", + 1: "primary reference", + } + """stratum table""" + + MODE_TABLE = { + 0: "unspecified", + 1: "symmetric active", + 2: "symmetric passive", + 3: "client", + 4: "server", + 5: "broadcast", + 6: "reserved for NTP control messages", + 7: "reserved for private use", + } + """mode table""" + + LEAP_TABLE = { + 0: "no warning", + 1: "last minute has 61 seconds", + 2: "last minute has 59 seconds", + 3: "alarm condition (clock not synchronized)", + } + """leap indicator table""" class NTPPacket: - """NTP packet class. + """NTP packet class. + + This represents an NTP packet. + """ + + _PACKET_FORMAT = "!B B B b 11I" + """packet format to pack/unpack""" - This represents an NTP packet. + def __init__(self, version=4, mode=3, tx_timestamp=0): + """Constructor. + + Parameters: + version -- NTP version + mode -- packet mode (client, server) + tx_timestamp -- packet transmit timestamp """ - - _PACKET_FORMAT = "!B B B b 11I" - """packet format to pack/unpack""" - - def __init__(self, version=4, mode=3, tx_timestamp=0): - """Constructor. - - Parameters: - version -- NTP version - mode -- packet mode (client, server) - tx_timestamp -- packet transmit timestamp - """ - self.leap = 0 - """leap second indicator""" - self.version = version - """version""" - self.mode = mode - """mode""" - self.stratum = 0 - """stratum""" - self.poll = 0 - """poll interval""" - self.precision = 0 - """precision""" - self.root_delay = 0 - """root delay""" - self.root_dispersion = 0 - """root dispersion""" - self.ref_id = 0 - """reference clock identifier""" - self.ref_timestamp = 0 - """reference timestamp""" - self.orig_timestamp = 0 - self.orig_timestamp_high = 0 - self.orig_timestamp_low = 0 - """originate timestamp""" - self.recv_timestamp = 0 - """receive timestamp""" - self.tx_timestamp = tx_timestamp - self.tx_timestamp_high = 0 - self.tx_timestamp_low = 0 - """tansmit timestamp""" - - def to_data(self): - """Convert this NTPPacket to a buffer that can be sent over a socket. - - Returns: - buffer representing this packet - - Raises: - NTPException -- in case of invalid field - """ - try: - packed = struct.pack(NTPPacket._PACKET_FORMAT, - (self.leap << 6 | self.version << 3 | self.mode), - self.stratum, - self.poll, - self.precision, - _to_int(self.root_delay) << 16 | _to_frac(self.root_delay, 16), - _to_int(self.root_dispersion) << 16 | - _to_frac(self.root_dispersion, 16), - self.ref_id, - _to_int(self.ref_timestamp), - _to_frac(self.ref_timestamp), - #Change by lichen, avoid loss of precision - self.orig_timestamp_high, - self.orig_timestamp_low, - _to_int(self.recv_timestamp), - _to_frac(self.recv_timestamp), - _to_int(self.tx_timestamp), - _to_frac(self.tx_timestamp)) - except struct.error: - raise NTPException("Invalid NTP packet fields.") - return packed - - def from_data(self, data): - """Populate this instance from a NTP packet payload received from - the network. - - Parameters: - data -- buffer payload - - Raises: - NTPException -- in case of invalid packet format - """ - try: - unpacked = struct.unpack(NTPPacket._PACKET_FORMAT, - data[0:struct.calcsize(NTPPacket._PACKET_FORMAT)]) - except struct.error: - raise NTPException("Invalid NTP packet.") - - self.leap = unpacked[0] >> 6 & 0x3 - self.version = unpacked[0] >> 3 & 0x7 - self.mode = unpacked[0] & 0x7 - self.stratum = unpacked[1] - self.poll = unpacked[2] - self.precision = unpacked[3] - self.root_delay = float(unpacked[4])/2**16 - self.root_dispersion = float(unpacked[5])/2**16 - self.ref_id = unpacked[6] - self.ref_timestamp = _to_time(unpacked[7], unpacked[8]) - self.orig_timestamp = _to_time(unpacked[9], unpacked[10]) - self.orig_timestamp_high = unpacked[9] - self.orig_timestamp_low = unpacked[10] - self.recv_timestamp = _to_time(unpacked[11], unpacked[12]) - self.tx_timestamp = _to_time(unpacked[13], unpacked[14]) - self.tx_timestamp_high = unpacked[13] - self.tx_timestamp_low = unpacked[14] - - def GetTxTimeStamp(self): - return (self.tx_timestamp_high,self.tx_timestamp_low) - - def SetOriginTimeStamp(self,high,low): - self.orig_timestamp_high = high - self.orig_timestamp_low = low - + self.leap = 0 + """leap second indicator""" + self.version = version + """version""" + self.mode = mode + """mode""" + self.stratum = 0 + """stratum""" + self.poll = 0 + """poll interval""" + self.precision = 0 + """precision""" + self.root_delay = 0 + """root delay""" + self.root_dispersion = 0 + """root dispersion""" + self.ref_id = 0 + """reference clock identifier""" + self.ref_timestamp = 0 + """reference timestamp""" + self.orig_timestamp = 0 + self.orig_timestamp_high = 0 + self.orig_timestamp_low = 0 + """originate timestamp""" + self.recv_timestamp = 0 + """receive timestamp""" + self.tx_timestamp = tx_timestamp + self.tx_timestamp_high = 0 + self.tx_timestamp_low = 0 + """tansmit timestamp""" + + def to_data(self): + """Convert this NTPPacket to a buffer that can be sent over a socket. + + Returns: + buffer representing this packet + + Raises: + NTPException -- in case of invalid field + """ + try: + packed = struct.pack(NTPPacket._PACKET_FORMAT, + (self.leap << 6 | self.version << 3 | self.mode), + self.stratum, + self.poll, + self.precision, + _to_int(self.root_delay) << 16 | _to_frac(self.root_delay, 16), + _to_int(self.root_dispersion) << 16 | + _to_frac(self.root_dispersion, 16), + self.ref_id, + _to_int(self.ref_timestamp), + _to_frac(self.ref_timestamp), + #Change by lichen, avoid loss of precision + self.orig_timestamp_high, + self.orig_timestamp_low, + _to_int(self.recv_timestamp), + _to_frac(self.recv_timestamp), + _to_int(self.tx_timestamp), + _to_frac(self.tx_timestamp)) + except struct.error: + raise NTPException("Invalid NTP packet fields.") + return packed + + def from_data(self, data): + """Populate this instance from a NTP packet payload received from + the network. + + Parameters: + data -- buffer payload + + Raises: + NTPException -- in case of invalid packet format + """ + try: + unpacked = struct.unpack(NTPPacket._PACKET_FORMAT, + data[0:struct.calcsize(NTPPacket._PACKET_FORMAT)]) + except struct.error: + raise NTPException("Invalid NTP packet.") + + self.leap = unpacked[0] >> 6 & 0x3 + self.version = unpacked[0] >> 3 & 0x7 + self.mode = unpacked[0] & 0x7 + self.stratum = unpacked[1] + self.poll = unpacked[2] + self.precision = unpacked[3] + self.root_delay = float(unpacked[4])/2**16 + self.root_dispersion = float(unpacked[5])/2**16 + self.ref_id = unpacked[6] + self.ref_timestamp = _to_time(unpacked[7], unpacked[8]) + self.orig_timestamp = _to_time(unpacked[9], unpacked[10]) + self.orig_timestamp_high = unpacked[9] + self.orig_timestamp_low = unpacked[10] + self.recv_timestamp = _to_time(unpacked[11], unpacked[12]) + self.tx_timestamp = _to_time(unpacked[13], unpacked[14]) + self.tx_timestamp_high = unpacked[13] + self.tx_timestamp_low = unpacked[14] + + def GetTxTimeStamp(self): + return (self.tx_timestamp_high,self.tx_timestamp_low) + + def SetOriginTimeStamp(self,high,low): + self.orig_timestamp_high = high + self.orig_timestamp_low = low class RecvThread(threading.Thread): - def __init__(self,socket): - threading.Thread.__init__(self) - self.socket = socket - def run(self): - global t,stopFlag - while True: - if stopFlag == True: - print("RecvThread Ended") - break - rlist,wlist,elist = select.select([self.socket],[],[],1); - if len(rlist) != 0: - print("Received %d packets" % len(rlist)) - for tempSocket in rlist: - try: - data,addr = tempSocket.recvfrom(1024) - recvTimestamp = recvTimestamp = system_to_ntp_time(time.time()) - taskQueue.put((data,addr,recvTimestamp)) - except socket.error as msg: - print(msg) + + def __init__(self,socket): + threading.Thread.__init__(self) + self.socket = socket + + def run(self): + global t,stop_flag + while True: + if stop_flag == True: + print("RecvThread Ended") + break + rlist,wlist,elist = select.select([self.socket],[],[],1) + if len(rlist) != 0: + print("Received %d packets" % len(rlist)) + for tempSocket in rlist: + try: + data,addr = tempSocket.recvfrom(1024) + recvTimestamp = recvTimestamp = system_to_ntp_time(time.time()) + taskQueue.put((data,addr,recvTimestamp)) + except socket.error as msg: + print(msg) class WorkThread(threading.Thread): - def __init__(self,socket): - threading.Thread.__init__(self) - self.socket = socket - def run(self): - global taskQueue,stopFlag - while True: - if stopFlag == True: - print("WorkThread Ended") - break - try: - data,addr,recvTimestamp = taskQueue.get(timeout=1) - recvPacket = NTPPacket() - recvPacket.from_data(data) - timeStamp_high,timeStamp_low = recvPacket.GetTxTimeStamp() - sendPacket = NTPPacket(version=4,mode=4) - sendPacket.stratum = 2 - sendPacket.poll = 10 - ''' - sendPacket.precision = 0xfa - sendPacket.root_delay = 0x0bfa - sendPacket.root_dispersion = 0x0aa7 - sendPacket.ref_id = 0x808a8c2c - ''' - sendPacket.ref_timestamp = recvTimestamp-5 - sendPacket.SetOriginTimeStamp(timeStamp_high,timeStamp_low) - sendPacket.recv_timestamp = recvTimestamp - sendPacket.tx_timestamp = system_to_ntp_time(time.time()) - socket.sendto(sendPacket.to_data(),addr) - print("Sent to %s:%d" % (addr[0],addr[1])) - except queue.Empty: - continue - - -listenIp = "0.0.0.0" -listenPort = 123 + + def __init__(self,socket): + threading.Thread.__init__(self) + self.socket = socket + + def run(self): + global taskQueue,stop_flag + while True: + if stop_flag is True: + print("WorkThread Ended") + break + try: + data,addr,recvTimestamp = taskQueue.get(timeout=1) + recvPacket = NTPPacket() + recvPacket.from_data(data) + timeStamp_high,timeStamp_low = recvPacket.GetTxTimeStamp() + sendPacket = NTPPacket(version=4,mode=4) + sendPacket.stratum = 2 + sendPacket.poll = 10 + sendPacket.ref_timestamp = recvTimestamp-5 + sendPacket.SetOriginTimeStamp(timeStamp_high,timeStamp_low) + sendPacket.recv_timestamp = recvTimestamp + sendPacket.tx_timestamp = system_to_ntp_time(time.time()) + socket.sendto(sendPacket.to_data(),addr) + print("Sent to %s:%d" % (addr[0],addr[1])) + except queue.Empty: + continue + +listen_ip = "0.0.0.0" +listen_port = 123 socket = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) -socket.bind((listenIp,listenPort)) -print("local socket: ", socket.getsockname()); +socket.bind((listen_ip,listen_port)) +print(f"local socket: {socket.getsockname()}") recvThread = RecvThread(socket) recvThread.start() workThread = WorkThread(socket) workThread.start() while True: - try: - time.sleep(0.5) - except KeyboardInterrupt: - print("Exiting...") - stopFlag = True - recvThread.join() - workThread.join() - #socket.close() - print("Exited") - break - + try: + time.sleep(0.5) + except KeyboardInterrupt: + print("Exiting...") + stop_flag = True + recvThread.join() + workThread.join() + #socket.close() + print("Exited") + break diff --git a/net_orc/network/modules/radius/conf/ca.crt b/net_orc/network/modules/radius/conf/ca.crt index d009cb1ab..bb8aadf6a 100644 --- a/net_orc/network/modules/radius/conf/ca.crt +++ b/net_orc/network/modules/radius/conf/ca.crt @@ -1,26 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIEYTCCA0mgAwIBAgIUQJ4F8hBCnCp7ASPZqG/tNQgoUR4wDQYJKoZIhvcNAQEL -BQAwgb8xCzAJBgNVBAYTAkdCMRswGQYDVQQIDBIbWzN+TGVpY2VzdGVyc2hpcmUx -FTATBgNVBAcMDExvdWdoYm9yb3VnaDEUMBIGA1UECgwLRm9yZXN0IFJvY2sxDjAM -BgNVBAsMBUN5YmVyMR8wHQYDVQQDDBZjeWJlci5mb3Jlc3Ryb2NrLmNvLnVrMTUw -MwYJKoZIhvcNAQkBFiZjeWJlcnNlY3VyaXR5LnRlc3RpbmdAZm9yZXN0cm9jay5j -by51azAeFw0yMjAzMDQxMjEzMTBaFw0yNzAzMDMxMjEzMTBaMIG/MQswCQYDVQQG -EwJHQjEbMBkGA1UECAwSG1szfkxlaWNlc3RlcnNoaXJlMRUwEwYDVQQHDAxMb3Vn -aGJvcm91Z2gxFDASBgNVBAoMC0ZvcmVzdCBSb2NrMQ4wDAYDVQQLDAVDeWJlcjEf -MB0GA1UEAwwWY3liZXIuZm9yZXN0cm9jay5jby51azE1MDMGCSqGSIb3DQEJARYm -Y3liZXJzZWN1cml0eS50ZXN0aW5nQGZvcmVzdHJvY2suY28udWswggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDNz3vJiZ5nX8lohEhqXvxEme3srip8qF7 -r5ScIeQzsTKuPNAmoefx9TcU3SyA2BnREuDX+OCYMN62xxWG2PndOl0LNezAY22C -PJwHbaBntLKY/ZhxYSTyratM7zxKSVLtClamA/bJXBhdfZZKYOP3xlZQEQTygtzK -j5hZwDrpDARtjRZIMWPLqVcoaW9ow2urJVsdD4lYAhpQU2UIgiWo7BG3hJsUfcYX -EQyyrMKJ7xaCwzIU7Sem1PETrzeiWg4KhDijc7A0RMPWlU5ljf0CnY/IZwiDsMRl -hGmGBPvR+ddiWPZPtSKj6TPWpsaMUR9UwncLmSSrhf1otX4Mw0vbAgMBAAGjUzBR -MB0GA1UdDgQWBBR0Qxx2mDTPIfpnzO5YtycGs6t8ijAfBgNVHSMEGDAWgBR0Qxx2 -mDTPIfpnzO5YtycGs6t8ijAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA -A4IBAQCpTMBMZGXF74WCxrIk23MUsu0OKzMs8B16Wy8BHz+7hInLZwbkx71Z0TP5 -rsMITetSANtM/k4jH7Vmr1xmzU7oSz5zKU1+7rIjKjGtih48WZdJay0uqfKe0K2s -vsRS0LVLY6IiTFWK9YrLC0QFSK7z5GDl1oc/D5yIZAkbsL6PRQJ5RQsYf5BhHfyB -PRV/KcF7c9iKVYW2vILJzbyYLHTDADTHbtfCe5+pAGxagswDjSMVkQu5iJNjbtUO -5iv7PRkgzUFru9Kk6q+LrXbzyPPCwlc3Xbh1q5jSkJLkcV3K26E7+uX5HI+Hxpeh -a8kOsdnw+N8wX6bc7eXIaGBDMine ------END CERTIFICATE----- +MIIFDzCCAvegAwIBAgIJAOb7lZzENM1TMA0GCSqGSIb3DQEBCwUAMB0xCzAJBgNV +BAYTAkZSMQ4wDAYDVQQKDAVDQVRUSzAgFw0yMjEwMDcwODIxNTVaGA8yMDcyMDky +NDA4MjE1NVowHTELMAkGA1UEBhMCRlIxDjAMBgNVBAoMBUNBVFRLMIICIjANBgkq +hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsZ+wd41TfLs5Vh5Wz1ESqIxwzu3iHWjJ +KbOlpnPdI6uPo9DU5xdmhcH0jc/RVis+EVn1ylFyzN3l4uIACah1Dk3frFXN/LWc +EzN7DyyHO56HZ5IpOFazVMQn5xrRwsglRop6et+Azqm+3xDpBSoKg8YhBAUsezuT +N0XlpsN3BMLjVXfwrTV1ECKP0Emg3qP3EaKRm1EdQ0uVNRNe24q5EDWiLnqlD14a +X5w1hHAj0Rr9kmKo+fs9WL7vIzbgy6xccfkKE8Wk7IR/xabTNjC5x+/7Pscqthic +tGYQ+Rm4Z1XTYDKBgoFHdI2ouscmiceqxESu3hW/IBe3iLin84kGywRGrzjLcOFI +adAj+0y3lGGV7Vw2RI3bUA6oOM8V1zbFUsZLq6+ylmvw0HQLAUeBODo6Iwu8ACxT +8/A+LmBUZFk7copLfvqFUmt8vjP7XiDuYsGvVJrTc6MJWWOITqyirhAkcP/vPoNK +l8PXhLGo66xG+hC57gCm3d3IwkXNLW6UhCHIuUa6LTTTaTehy2unDEm7Rt4ghWlw +2JuDr7QcZrWrRj1OwVAiPNkjLCF30aKxnVQxc2JY9W3H+xRC0YlDNmOpdHHvuJfS +1y1tNUq+fZQGybubDsa0l0LHfoKRGfeFXnxT6tyvNnGEaJG9mkLPXPkEBuadrnvA +oZeymb/D440CAwEAAaNQME4wHQYDVR0OBBYEFHKNGWOtO3haPEkZSVfgnxbEbTs3 +MB8GA1UdIwQYMBaAFHKNGWOtO3haPEkZSVfgnxbEbTs3MAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggIBAGzuawd3tYwDCnO84eZ+NT0uqQk37ss1fdwgDY7G +dAaeSb0oW5+tejdiLzWsKgHL7g6ClEOS3f2Vvo65fsH7BA5Ppmhk+4U6ppqbBiGG +v5WqnRz7OpdMTqunFsmYhbgbu+OZX3TTg6hkIYxFHlHDMpcMwMYtlWsPRZSlTM09 +BbaWyhqTZjbUIxemwc3JW+5zRYoA2ii/Om/2/9iUbngVqEilmUrflMcfn81ddate +0XwMcm/qhyKU+CIAPXmmtLkTms66FSSXMfqy1HizzSsCFntozUA7mtPRm53IsGpR +TOdGTe5Y5jJ/dlXwmZ5dmWBR8qlyxLpG0iB7KWNxs+V7B6kCFU3BhiLPiS/BnDap +EE1JDKu1jktJhxeAhmSsrvZ10bCKZW+dQbSjqr3wScYok/f05daB97LaAs869jra +93uJ7dYA9gfUtkaqZW9oqPrIO3FNZLL5D1z6eWcGC2+3MLhrtNTov3fthFGJyWf7 +iCBdQYofeR4EA4nfI+QcM2HAHNtChGESZ/8p/eBSU4GQW7zURELIKJ5OeTJZGAgs +bMbNbqbiyzCSuM2CHTN+Nw0rMc9AXkqSV57scCu/2ui1z1GKWeI65hKhwc++IXP7 +lJWv710T4+9DOgoi5sFNNLbRcVmkUeodFje83PTs+U/hgvQHW1+RTJ4ESTPMqVf1 +VTyk +-----END CERTIFICATE----- \ No newline at end of file diff --git a/net_orc/python/src/network_orchestrator.py b/net_orc/python/src/network_orchestrator.py index 39fd3339c..53a94b795 100644 --- a/net_orc/python/src/network_orchestrator.py +++ b/net_orc/python/src/network_orchestrator.py @@ -535,7 +535,7 @@ def start_network_services(self): LOGGER.info('All network services are running') self._check_network_services() - def _attach_test_module_to_network(self, test_module): + def attach_test_module_to_network(self, test_module): LOGGER.debug('Attaching test module ' + test_module.display_name + ' to device bridge') diff --git a/test_orc/modules/dns/python/src/dns_module.py b/test_orc/modules/dns/python/src/dns_module.py index f1333ce14..49c2b5fe9 100644 --- a/test_orc/modules/dns/python/src/dns_module.py +++ b/test_orc/modules/dns/python/src/dns_module.py @@ -9,69 +9,69 @@ class DNSModule(TestModule): - def __init__(self, module): - super().__init__(module_name=module, log_name=LOG_NAME) - self._dns_server = "10.10.10.4" - global LOGGER - LOGGER = self._get_logger() - - def _check_dns_traffic(self, tcpdump_filter): - to_dns = self._exec_tcpdump(tcpdump_filter) - num_query_dns = len(to_dns) - LOGGER.info("DNS queries found: " + str(num_query_dns)) - dns_traffic_detected = len(to_dns) > 0 - LOGGER.info("DNS traffic detected: " + str(dns_traffic_detected)) - return dns_traffic_detected - - def _dns_network_from_dhcp(self): - LOGGER.info( - "Checking DNS traffic for configured DHCP DNS server: " + self._dns_server) - - # Check if the device DNS traffic is to appropriate server - tcpdump_filter = 'dst port 53 and dst host {} and ether src {}'.format( - self._dns_server, self._device_mac) - - result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) - - LOGGER.info( - "DNS traffic detected to configured DHCP DNS server: " + str(result)) - return result - - def _dns_network_from_device(self): - LOGGER.info("Checking DNS traffic from device: " + self._device_mac) - - # Check if the device DNS traffic is to appropriate server - tcpdump_filter = 'dst port 53 and ether src {}'.format( - self._device_mac) - - result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) - - LOGGER.info("DNS traffic detected from device: " + str(result)) - return result - - def _exec_tcpdump(self, tcpdump_filter): - """ - Args - tcpdump_filter: Filter to pass onto tcpdump file - capture_file: Optional capture file to look - Returns - List of packets matching the filter - """ - command = 'tcpdump -tttt -n -r {} {}'.format( - CAPTURE_FILE, tcpdump_filter) - - LOGGER.debug("tcpdump command: " + command) - - process = subprocess.Popen(command, - universal_newlines=True, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - text = str(process.stdout.read()).rstrip() - - LOGGER.debug("tcpdump response: " + text) - - if text: - return text.split("\n") - - return [] + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + self._dns_server = "10.10.10.4" + global LOGGER + LOGGER = self._get_logger() + + def _check_dns_traffic(self, tcpdump_filter): + to_dns = self._exec_tcpdump(tcpdump_filter) + num_query_dns = len(to_dns) + LOGGER.info("DNS queries found: " + str(num_query_dns)) + dns_traffic_detected = len(to_dns) > 0 + LOGGER.info("DNS traffic detected: " + str(dns_traffic_detected)) + return dns_traffic_detected + + def _dns_network_from_dhcp(self): + LOGGER.info( + "Checking DNS traffic for configured DHCP DNS server: " + self._dns_server) + + # Check if the device DNS traffic is to appropriate server + tcpdump_filter = "dst port 53 and dst host {} and ether src {}".format( + self._dns_server, self._device_mac) + + result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) + + LOGGER.info( + "DNS traffic detected to configured DHCP DNS server: " + str(result)) + return result + + def _dns_network_from_device(self): + LOGGER.info("Checking DNS traffic from device: " + self._device_mac) + + # Check if the device DNS traffic is to appropriate server + tcpdump_filter = "dst port 53 and ether src {}".format( + self._device_mac) + + result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) + + LOGGER.info("DNS traffic detected from device: " + str(result)) + return result + + def _exec_tcpdump(self, tcpdump_filter): + """ + Args + tcpdump_filter: Filter to pass onto tcpdump file + capture_file: Optional capture file to look + Returns + List of packets matching the filter + """ + command = "tcpdump -tttt -n -r {} {}".format( + CAPTURE_FILE, tcpdump_filter) + + LOGGER.debug("tcpdump command: " + command) + + process = subprocess.Popen(command, + universal_newlines=True, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + text = str(process.stdout.read()).rstrip() + + LOGGER.debug("tcpdump response: " + text) + + if text: + return text.split("\n") + + return [] diff --git a/test_orc/modules/dns/python/src/run.py b/test_orc/modules/dns/python/src/run.py index e5fedb67b..e09350e6c 100644 --- a/test_orc/modules/dns/python/src/run.py +++ b/test_orc/modules/dns/python/src/run.py @@ -4,7 +4,6 @@ import signal import sys import logger -import time from dns_module import DNSModule @@ -14,45 +13,47 @@ class DNSModuleRunner: - def __init__(self,module): + def __init__(self,module): - signal.signal(signal.SIGINT, self._handler) - signal.signal(signal.SIGTERM, self._handler) - signal.signal(signal.SIGABRT, self._handler) - signal.signal(signal.SIGQUIT, self._handler) - self.add_logger(module) + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) + self.add_logger(module) - LOGGER.info("Starting DNS Test Module") + LOGGER.info("Starting DNS Test Module") - self._test_module = DNSModule(module) - self._test_module.run_tests() + self._test_module = DNSModule(module) + self._test_module.run_tests() - LOGGER.info("DNS Test Module Finished") + LOGGER.info("DNS Test Module Finished") - def add_logger(self, module): - global LOGGER - LOGGER = logger.get_logger(LOG_NAME, module) + def add_logger(self, module): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, module) - def _handler(self, signum, *other): - LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received. Stopping test module...") - LOGGER.info("Test module stopped") - sys.exit(1) + def _handler(self, signum, *other): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping test module...") + LOGGER.info("Test module stopped") + sys.exit(1) def run(argv): - parser = argparse.ArgumentParser(description="Test Module DNS", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description="Test Module DNS", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - "-m", "--module", help="Define the module name to be used to create the log file") + parser.add_argument( + "-m", + "--module", + help="Define the module name to be used to create the log file") - args = parser.parse_args() + args = parser.parse_args() - # For some reason passing in the args from bash adds an extra - # space before the argument so we'll just strip out extra space - DNSModuleRunner(args.module.strip()) + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + DNSModuleRunner(args.module.strip()) if __name__ == "__main__": - run(sys.argv) + run(sys.argv) diff --git a/test_orc/modules/nmap/python/src/nmap_module.py b/test_orc/modules/nmap/python/src/nmap_module.py index 7d5bd3604..477dbe3c6 100644 --- a/test_orc/modules/nmap/python/src/nmap_module.py +++ b/test_orc/modules/nmap/python/src/nmap_module.py @@ -12,216 +12,220 @@ class NmapModule(TestModule): - def __init__(self, module): - super().__init__(module_name=module, log_name=LOG_NAME) - self._unallowed_ports = [] - self._scan_tcp_results = None - self._udp_tcp_results = None - self._script_scan_results = None - global LOGGER - LOGGER = self._get_logger() - - def _security_nmap_ports(self, config): - LOGGER.info( - "Running security.nmap.ports test") - - # Delete the enabled key from the config if it exists - # to prevent it being treated as a test key - if "enabled" in config: - del config["enabled"] - - if self._device_ipv4_addr is not None: - # Run the monitor method asynchronously to keep this method non-blocking - self._tcp_scan_thread = threading.Thread( - target=self._scan_tcp_ports, args=(config,)) - self._udp_scan_thread = threading.Thread( - target=self._scan_udp_ports, args=(config,)) - self._script_scan_thread = threading.Thread( - target=self._scan_scripts, args=(config,)) - - self._tcp_scan_thread.daemon = True - self._udp_scan_thread.daemon = True - self._script_scan_thread.daemon = True - - self._tcp_scan_thread.start() - self._udp_scan_thread.start() - self._script_scan_thread.start() - - while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive() or self._script_scan_thread.is_alive(): - time.sleep(1) - - LOGGER.debug("TCP scan results: " + str(self._scan_tcp_results)) - LOGGER.debug("UDP scan results: " + str(self._scan_udp_results)) - LOGGER.debug("Service scan results: " + - str(self._script_scan_results)) - self._process_port_results( - tests=config) - LOGGER.info("Unallowed Ports: " + str(self._unallowed_ports)) - LOGGER.info("Script scan results:\n" + - json.dumps(self._script_scan_results)) - return len(self._unallowed_ports) == 0 + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + self._unallowed_ports = [] + self._scan_tcp_results = None + self._udp_tcp_results = None + self._script_scan_results = None + global LOGGER + LOGGER = self._get_logger() + + def _security_nmap_ports(self, config): + LOGGER.info( + "Running security.nmap.ports test") + + # Delete the enabled key from the config if it exists + # to prevent it being treated as a test key + if "enabled" in config: + del config["enabled"] + + if self._device_ipv4_addr is not None: + # Run the monitor method asynchronously to keep this method non-blocking + self._tcp_scan_thread = threading.Thread( + target=self._scan_tcp_ports, args=(config,)) + self._udp_scan_thread = threading.Thread( + target=self._scan_udp_ports, args=(config,)) + self._script_scan_thread = threading.Thread( + target=self._scan_scripts, args=(config,)) + + self._tcp_scan_thread.daemon = True + self._udp_scan_thread.daemon = True + self._script_scan_thread.daemon = True + + self._tcp_scan_thread.start() + self._udp_scan_thread.start() + self._script_scan_thread.start() + + while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive() or self._script_scan_thread.is_alive(): + time.sleep(1) + + LOGGER.debug("TCP scan results: " + str(self._scan_tcp_results)) + LOGGER.debug("UDP scan results: " + str(self._scan_udp_results)) + LOGGER.debug("Service scan results: " + + str(self._script_scan_results)) + self._process_port_results( + tests=config) + LOGGER.info("Unallowed Ports: " + str(self._unallowed_ports)) + LOGGER.info("Script scan results:\n" + + json.dumps(self._script_scan_results)) + return len(self._unallowed_ports) == 0 + else: + LOGGER.info("Device ip address not resolved, skipping") + return None + + def _process_port_results(self, tests): + for test in tests: + LOGGER.info("Checking results for test: " + str(test)) + self._check_scan_results(test_config=tests[test]) + + def _check_scan_results(self, test_config): + port_config = {} + if "tcp_ports" in test_config: + port_config.update(test_config["tcp_ports"]) + elif "udp_ports" in test_config: + port_config.update(test_config["udp_ports"]) + + scan_results = {} + if self._scan_tcp_results is not None: + scan_results.update(self._scan_tcp_results) + if self._scan_udp_results is not None: + scan_results.update(self._scan_udp_results) + if self._script_scan_results is not None: + scan_results.update(self._script_scan_results) + if port_config is not None: + for port in port_config: + result = None + LOGGER.info("Checking port: " + str(port)) + LOGGER.debug("Port config: " + str(port_config[port])) + if port in scan_results: + if scan_results[port]["state"] == "open": + if not port_config[port]["allowed"]: + LOGGER.info("Unallowed port open") + self._unallowed_ports.append(str(port)) + result = False + else: + LOGGER.info("Allowed port open") + result = True + else: + LOGGER.info("Port is closed") + result = True else: - LOGGER.info("Device ip address not resolved, skipping") - return None - - def _process_port_results(self, tests): - for test in tests: - LOGGER.info("Checking results for test: " + str(test)) - self._check_scan_results(test_config=tests[test]) - - def _check_scan_results(self, test_config): - port_config = {} - if "tcp_ports" in test_config: - port_config.update(test_config["tcp_ports"]) - elif "udp_ports" in test_config: - port_config.update(test_config["udp_ports"]) - - scan_results = {} - if self._scan_tcp_results is not None: - scan_results.update(self._scan_tcp_results) - if self._scan_udp_results is not None: - scan_results.update(self._scan_udp_results) - if self._script_scan_results is not None: - scan_results.update(self._script_scan_results) - if port_config is not None: - for port in port_config: - result = None - LOGGER.info("Checking port: " + str(port)) - LOGGER.debug("Port config: " + str(port_config[port])) - if port in scan_results: - if scan_results[port]["state"] == "open": - if not port_config[port]["allowed"]: - LOGGER.info("Unallowed port open") - self._unallowed_ports.append(str(port)) - result = False - else: - LOGGER.info("Allowed port open") - result = True - else: - LOGGER.info("Port is closed") - result = True - else: - LOGGER.info("Port not detected, closed") - result = True - - if result is not None: - port_config[port]["result"] = "compliant" if result else "non-compliant" - else: - port_config[port]["result"] = "skipped" - - def _scan_scripts(self, tests): - scan_results = {} - LOGGER.info("Checing for scan scripts") - for test in tests: - test_config = tests[test] - if "tcp_ports" in test_config: - for port in test_config["tcp_ports"]: - port_config = test_config["tcp_ports"][port] - if "service_scan" in port_config: - LOGGER.info("Service Scan Detected for: " + str(port)) - svc = port_config["service_scan"] - scan_results.update( - self._scan_tcp_with_script(svc["script"])) - if "udp_ports" in test_config: - for port in test_config["udp_ports"]: - if "service_scan" in port: - LOGGER.info("Service Scan Detected for: " + str(port)) - svc = port["service_scan"] - self._scan_udp_with_script(svc["script"], port) - scan_results.update( - self._scan_tcp_with_script(svc["script"])) - self._script_scan_results = scan_results - - def _scan_tcp_with_script(self, script_name, ports=None): - LOGGER.info("Running TCP nmap scan with script " + script_name) - scan_options = " -v -n T3 --host-timeout=6m -A --script " + script_name - port_options = " --open " - if ports is None: - port_options += " -p- " - else: - port_options += " -p" + ports + " " - results_file = "/runtime/output/" + self._module_name + "-"+script_name+".log" - nmap_options = scan_options + port_options + " -oG " + results_file - nmap_results, err = util.run_command( - "nmap " + nmap_options + " " + self._device_ipv4_addr) - LOGGER.info("Nmap TCP script scan complete") - LOGGER.info("nmap script results\n" + str(nmap_results)) - return self._process_nmap_results(nmap_results=nmap_results) - - def _scan_udp_with_script(self, script_name, ports=None): - LOGGER.info("Running UDP nmap scan with script " + script_name) - scan_options = " --sU -Pn -n --script " + script_name - port_options = " --open " - if ports is None: - port_options += " -p- " + LOGGER.info("Port not detected, closed") + result = True + + if result is not None: + port_config[port]["result"] = "compliant" if result else "non-compliant" else: - port_options += " -p" + ports + " " - nmap_options = scan_options + port_options - nmap_results, err = util.run_command( - "nmap " + nmap_options + self._device_ipv4_addr) - LOGGER.info("Nmap UDP script scan complete") - LOGGER.info("nmap script results\n" + str(nmap_results)) - return self._process_nmap_results(nmap_results=nmap_results) - - def _scan_tcp_ports(self, tests): - max_port = 1000 - ports = [] - for test in tests: - test_config = tests[test] - if "tcp_ports" in test_config: - for port in test_config["tcp_ports"]: - if int(port) > max_port: - ports.append(port) - ports_to_scan = "1-" + str(max_port) - if len(ports) > 0: - ports_to_scan += "," + ','.join(ports) - LOGGER.info("Running nmap TCP port scan") - LOGGER.info("TCP ports: " + str(ports_to_scan)) - nmap_results, err = util.run_command( - "nmap -sT -sV -Pn -v -p " + ports_to_scan + " --version-intensity 7 -T4 " + self._device_ipv4_addr) - LOGGER.info("TCP port scan complete") - self._scan_tcp_results = self._process_nmap_results( - nmap_results=nmap_results) - - def _scan_udp_ports(self, tests): - ports = [] - for test in tests: - test_config = tests[test] - if "udp_ports" in test_config: - for port in test_config["udp_ports"]: - ports.append(port) - if len(ports) > 0: - port_list = ','.join(ports) - LOGGER.info("Running nmap UDP port scan") - LOGGER.info("UDP ports: " + str(port_list)) - nmap_results, err = util.run_command( - "nmap -sU -sV -p " + port_list + " " + self._device_ipv4_addr) - LOGGER.info("UDP port scan complete") - self._scan_udp_results = self._process_nmap_results( - nmap_results=nmap_results) - - def _process_nmap_results(self, nmap_results): - results = {} - LOGGER.info("nmap results\n" + str(nmap_results)) - if nmap_results: - if "Service Info" in nmap_results: - rows = nmap_results.split("PORT")[1].split( - "Service Info")[0].split("\n") - elif "PORT" in nmap_results: - rows = nmap_results.split("PORT")[1].split( - "MAC Address")[0].split("\n") - if rows: - for result in rows[1:-1]: # Iterate skipping the header and tail rows - cols = result.split() - port = cols[0].split("/")[0] - # If results don't start with a a port number, it's likely a bleed over - # from previous result so we need to ignore it - if port.isdigit(): - version = "" - if len(cols) > 3: - # recombine full version information that may contain spaces - version = ' '.join(cols[3:]) - port_result = {cols[0].split( - "/")[0]: {"state": cols[1], "service": cols[2], "version": version}} - results.update(port_result) - return results + port_config[port]["result"] = "skipped" + + def _scan_scripts(self, tests): + scan_results = {} + LOGGER.info("Checing for scan scripts") + for test in tests: + test_config = tests[test] + if "tcp_ports" in test_config: + for port in test_config["tcp_ports"]: + port_config = test_config["tcp_ports"][port] + if "service_scan" in port_config: + LOGGER.info("Service Scan Detected for: " + str(port)) + svc = port_config["service_scan"] + scan_results.update( + self._scan_tcp_with_script(svc["script"])) + if "udp_ports" in test_config: + for port in test_config["udp_ports"]: + if "service_scan" in port: + LOGGER.info("Service Scan Detected for: " + str(port)) + svc = port["service_scan"] + self._scan_udp_with_script(svc["script"], port) + scan_results.update( + self._scan_tcp_with_script(svc["script"])) + self._script_scan_results = scan_results + + def _scan_tcp_with_script(self, script_name, ports=None): + LOGGER.info("Running TCP nmap scan with script " + script_name) + scan_options = " -v -n T3 --host-timeout=6m -A --script " + script_name + port_options = " --open " + if ports is None: + port_options += " -p- " + else: + port_options += " -p" + ports + " " + results_file = f"/runtime/output/{self._module_name}-script_name.log" + nmap_options = scan_options + port_options + " -oG " + results_file + nmap_results, err = util.run_command( + "nmap " + nmap_options + " " + self._device_ipv4_addr) + LOGGER.info("Nmap TCP script scan complete") + LOGGER.info("nmap script results\n" + str(nmap_results)) + return self._process_nmap_results(nmap_results=nmap_results) + + def _scan_udp_with_script(self, script_name, ports=None): + LOGGER.info("Running UDP nmap scan with script " + script_name) + scan_options = " --sU -Pn -n --script " + script_name + port_options = " --open " + if ports is None: + port_options += " -p- " + else: + port_options += " -p" + ports + " " + nmap_options = scan_options + port_options + nmap_results = util.run_command( + "nmap " + nmap_options + self._device_ipv4_addr)[0] + LOGGER.info("Nmap UDP script scan complete") + LOGGER.info("nmap script results\n" + str(nmap_results)) + return self._process_nmap_results(nmap_results=nmap_results) + + def _scan_tcp_ports(self, tests): + max_port = 1000 + ports = [] + for test in tests: + test_config = tests[test] + if "tcp_ports" in test_config: + for port in test_config["tcp_ports"]: + if int(port) > max_port: + ports.append(port) + ports_to_scan = "1-" + str(max_port) + if len(ports) > 0: + ports_to_scan += "," + ",".join(ports) + LOGGER.info("Running nmap TCP port scan") + LOGGER.info("TCP ports: " + str(ports_to_scan)) + nmap_results = util.run_command( + f"""nmap -sT -sV -Pn -v -p {ports_to_scan} + --version-intensity 7 -T4 {self._device_ipv4_addr}""")[0] + LOGGER.info("TCP port scan complete") + self._scan_tcp_results = self._process_nmap_results( + nmap_results=nmap_results) + + def _scan_udp_ports(self, tests): + ports = [] + for test in tests: + test_config = tests[test] + if "udp_ports" in test_config: + for port in test_config["udp_ports"]: + ports.append(port) + if len(ports) > 0: + port_list = ",".join(ports) + LOGGER.info("Running nmap UDP port scan") + LOGGER.info("UDP ports: " + str(port_list)) + nmap_results = util.run_command( + f"nmap -sU -sV -p {port_list} {self._device_ipv4_addr}")[0] + LOGGER.info("UDP port scan complete") + self._scan_udp_results = self._process_nmap_results( + nmap_results=nmap_results) + + def _process_nmap_results(self, nmap_results): + results = {} + LOGGER.info("nmap results\n" + str(nmap_results)) + if nmap_results: + if "Service Info" in nmap_results: + rows = nmap_results.split("PORT")[1].split( + "Service Info")[0].split("\n") + elif "PORT" in nmap_results: + rows = nmap_results.split("PORT")[1].split( + "MAC Address")[0].split("\n") + if rows: + for result in rows[1:-1]: # Iterate skipping the header and tail rows + cols = result.split() + port = cols[0].split("/")[0] + # If results do not start with a a port number, + # it is likely a bleed over from previous result so + # we need to ignore it + if port.isdigit(): + version = "" + if len(cols) > 3: + # recombine full version information that may contain spaces + version = " ".join(cols[3:]) + port_result = {cols[0].split( + "/")[0]: {"state": cols[1], + "service": cols[2], + "version": version}} + results.update(port_result) + return results diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index f1e45e2f6..505fe1e49 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -51,8 +51,9 @@ def run_test_modules(self, device): self._run_test_module(module, device) LOGGER.info("All tests complete") LOGGER.info( - f"Completed running test modules on device with mac addr {device.mac_addr}") - results = self._generate_results(device) + f"""Completed running test modules on device + with mac addr {device.mac_addr}""") + self._generate_results(device) def _generate_results(self, device): results = {} @@ -65,21 +66,26 @@ def _generate_results(self, device): for module in self._test_modules: if module.enable_container and self._is_module_enabled(module,device): container_runtime_dir = os.path.join( - self._root_path, 'runtime/test/' + device.mac_addr.replace(':', '') + - '/' + module.name) - results_file = container_runtime_dir + '/' + module.name + '-result.json' + self._root_path, + "runtime/test/" + device.mac_addr.replace(":", "") + + "/" + module.name) + results_file = container_runtime_dir+"/"+module.name+"-result.json" try: - with open(results_file, 'r', encoding='UTF-8') as f: + with open(results_file, "r", encoding="UTF-8") as f: module_results = json.load(f) results[module.name] = module_results - except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: + except (FileNotFoundError, + PermissionError, + json.JSONDecodeError) as results_error: LOGGER.error("Module Results Errror " + module.name) LOGGER.debug(results_error) out_file = os.path.join( - self._root_path, 'runtime/test/' + device.mac_addr.replace(':', '') + '/results.json') - with open(out_file, 'w') as f: - json.dump(results,f,indent=2) + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + + "/results.json") + with open(out_file, "w", encoding="utf-8") as f: + json.dump(results,f,indent=2) return results def _is_module_enabled(self,module,device): @@ -87,7 +93,7 @@ def _is_module_enabled(self,module,device): if device.test_modules is not None: test_modules = json.loads(device.test_modules) if module.name in test_modules: - if 'enabled' in test_modules[module.name]: + if "enabled" in test_modules[module.name]: enabled = test_modules[module.name]["enabled"] return enabled @@ -122,10 +128,10 @@ def _run_test_module(self, module, device): mounts=[ Mount(target="/runtime/output", source=container_runtime_dir, - type='bind'), + type="bind"), Mount(target="/runtime/network", source=network_runtime_dir, - type='bind', + type="bind", read_only=True), ], environment={ @@ -144,13 +150,13 @@ def _run_test_module(self, module, device): # Mount the test container to the virtual network if requried if module.network: LOGGER.debug("Attaching test module to the network") - self._net_orc._attach_test_module_to_network(module) + self._net_orc.attach_test_module_to_network(module) # Determine the module timeout time test_module_timeout = time.time() + module.timeout status = self._get_module_status(module) - while time.time() < test_module_timeout and status == 'running': + while time.time() < test_module_timeout and status == "running": time.sleep(1) status = self._get_module_status(module) @@ -164,7 +170,9 @@ def _get_module_status(self, module): def _get_test_module(self, name): for test_module in self._test_modules: - if name == test_module.display_name or name == test_module.name or name == test_module.dir_name: + if name in [test_module.display_name, + test_module.name, + test_module.dir_name]: return test_module return None @@ -203,28 +211,28 @@ def _load_test_module(self, module_dir): # Load basic module information module = TestModule() with open(os.path.join(self._path, modules_dir, module_dir, MODULE_CONFIG), - encoding='UTF-8') as module_config_file: + encoding="UTF-8") as module_config_file: module_json = json.load(module_config_file) - module.name = module_json['config']['meta']['name'] - module.display_name = module_json['config']['meta']['display_name'] - module.description = module_json['config']['meta']['description'] + module.name = module_json["config"]["meta"]["name"] + module.display_name = module_json["config"]["meta"]["display_name"] + module.description = module_json["config"]["meta"]["description"] module.dir = os.path.join(self._path, modules_dir, module_dir) module.dir_name = module_dir module.build_file = module_dir + ".Dockerfile" module.container_name = "tr-ct-" + module.dir_name + "-test" module.image_name = "test-run/" + module.dir_name + "-test" - if 'timeout' in module_json['config']['docker']: - module.timeout = module_json['config']['docker']['timeout'] + if "timeout" in module_json["config"]["docker"]: + module.timeout = module_json["config"]["docker"]["timeout"] # Determine if this is a container or just an image/template - if "enable_container" in module_json['config']['docker']: - module.enable_container = module_json['config']['docker'][ - 'enable_container'] + if "enable_container" in module_json["config"]["docker"]: + module.enable_container = module_json["config"]["docker"][ + "enable_container"] - if "depends_on" in module_json['config']['docker']: - depends_on_module = module_json['config']['docker']['depends_on'] + if "depends_on" in module_json["config"]["docker"]: + depends_on_module = module_json["config"]["docker"]["depends_on"] if self._get_test_module(depends_on_module) is None: self._load_test_module(depends_on_module) diff --git a/testing/test_baseline.py b/testing/test_baseline.py index 3ab30a7c0..a53351c93 100644 --- a/testing/test_baseline.py +++ b/testing/test_baseline.py @@ -1,7 +1,7 @@ import json import pytest import re -import os +import os NTP_SERVER = '10.10.10.5' DNS_SERVER = '10.10.10.4' @@ -10,40 +10,43 @@ @pytest.fixture def container_data(): - dir = os.path.dirname(os.path.abspath(__file__)) - with open(CI_BASELINE_OUT) as f: - return json.load(f) + dir = os.path.dirname(os.path.abspath(__file__)) + with open(CI_BASELINE_OUT, encoding='utf-8') as f: + return json.load(f) @pytest.fixture def validator_results(): - dir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(dir, '../', 'runtime/validation/faux-dev/result.json')) as f: - return json.load(f) + dir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(dir, + '../', + 'runtime/validation/faux-dev/result.json'), + encoding='utf-8') as f: + return json.load(f) def test_internet_connectivity(container_data): - assert container_data['network']['internet'] == 200 + assert container_data['network']['internet'] == 200 def test_dhcp_ntp_option(container_data): - """ Check DHCP gives NTP server as option """ - assert container_data['dhcp']['ntp-servers'] == NTP_SERVER + """ Check DHCP gives NTP server as option """ + assert container_data['dhcp']['ntp-servers'] == NTP_SERVER def test_dhcp_dns_option(container_data): - assert container_data['dhcp']['domain-name-servers'] == DNS_SERVER + assert container_data['dhcp']['domain-name-servers'] == DNS_SERVER def test_assigned_ipv4_address(container_data): - assert int(container_data['network']['ipv4'].split('.')[-1][:-3]) > 10 + assert int(container_data['network']['ipv4'].split('.')[-1][:-3]) > 10 def test_ntp_server_reachable(container_data): - assert not 'no servers' in container_data['ntp_offset'] + assert not 'no servers' in container_data['ntp_offset'] def test_dns_server_reachable(container_data): - assert not 'no servers' in container_data['dns_response'] + assert not 'no servers' in container_data['dns_response'] def test_dns_server_resolves(container_data): - assert re.match(r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}', - container_data['dns_response']) + assert re.match(r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}', + container_data['dns_response']) def test_validator_results_compliant(validator_results): - results = [True if x['result'] == 'compliant' else False - for x in validator_results['results']] - assert all(results) + results = [True if x['result'] == 'compliant' else False + for x in validator_results['results']] + assert all(results) From 524c98fcc61ecc49b1a3ae9f2fcbd0c100961cfa Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 25 May 2023 11:48:33 +0100 Subject: [PATCH 02/12] More formatting fixes --- framework/logger.py | 49 +++++----- framework/test_runner.py | 93 +++++++++---------- test_orc/modules/base/python/src/logger.py | 49 +++++----- test_orc/modules/base/python/src/util.py | 30 +++--- .../baseline/python/src/baseline_module.py | 39 ++++---- test_orc/modules/baseline/python/src/run.py | 51 +++++----- 6 files changed, 155 insertions(+), 156 deletions(-) diff --git a/framework/logger.py b/framework/logger.py index 64d8fdb97..2442aea58 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -4,45 +4,46 @@ import os LOGGERS = {} -_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" +_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' _DATE_FORMAT = '%b %02d %H:%M:%S' _DEFAULT_LOG_LEVEL = logging.INFO _LOG_LEVEL = logging.INFO -_CONF_DIR = "conf" -_CONF_FILE_NAME = "system.json" -_LOG_DIR = "runtime/testing/" +_CONF_DIR = 'conf' +_CONF_FILE_NAME = 'system.json' +_LOG_DIR = 'runtime/testing/' # Set log level -with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as system_conf_file: - system_conf_json = json.load(system_conf_file) +with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), + encoding='utf-8') as system_conf_file: + system_conf_json = json.load(system_conf_file) log_level_str = system_conf_json['log_level'] temp_log = logging.getLogger('temp') try: - temp_log.setLevel(logging.getLevelName(log_level_str)) - _LOG_LEVEL = logging.getLevelName(log_level_str) + temp_log.setLevel(logging.getLevelName(log_level_str)) + _LOG_LEVEL = logging.getLevelName(log_level_str) except ValueError: - print('Invalid log level set in ' + _CONF_DIR + '/' + _CONF_FILE_NAME + - '. Using INFO as log level') - _LOG_LEVEL = _DEFAULT_LOG_LEVEL + print('Invalid log level set in ' + _CONF_DIR + '/' + _CONF_FILE_NAME + + '. Using INFO as log level') + _LOG_LEVEL = _DEFAULT_LOG_LEVEL log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) def add_file_handler(log, log_file): - handler = logging.FileHandler(_LOG_DIR + log_file + ".log") - handler.setFormatter(log_format) - log.addHandler(handler) + handler = logging.FileHandler(_LOG_DIR + log_file + '.log') + handler.setFormatter(log_format) + log.addHandler(handler) def add_stream_handler(log): - handler = logging.StreamHandler() - handler.setFormatter(log_format) - log.addHandler(handler) + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) def get_logger(name, log_file=None): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(_LOG_LEVEL) - add_stream_handler(LOGGERS[name]) - if log_file is not None: - add_file_handler(LOGGERS[name], log_file) - return LOGGERS[name] + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(_LOG_LEVEL) + add_stream_handler(LOGGERS[name]) + if log_file is not None: + add_file_handler(LOGGERS[name], log_file) + return LOGGERS[name] diff --git a/framework/test_runner.py b/framework/test_runner.py index 5c4bf1472..95f3e4208 100644 --- a/framework/test_runner.py +++ b/framework/test_runner.py @@ -14,61 +14,60 @@ import logger import signal -LOGGER = logger.get_logger('runner') - +LOGGER = logger.get_logger("runner") class TestRunner: + """Controls and starts the Test Run application.""" - def __init__(self, config_file=None, validate=True, net_only=False, single_intf=False): - self._register_exits() - self.test_run = TestRun(config_file=config_file, - validate=validate, - net_only=net_only, - single_intf=single_intf) - - def _register_exits(self): - signal.signal(signal.SIGINT, self._exit_handler) - signal.signal(signal.SIGTERM, self._exit_handler) - signal.signal(signal.SIGABRT, self._exit_handler) - signal.signal(signal.SIGQUIT, self._exit_handler) + def __init__(self, config_file=None, validate=True, + net_only=False, single_intf=False): + self._register_exits() + self.test_run = TestRun(config_file=config_file, + validate=validate, + net_only=net_only, + single_intf=single_intf) - def _exit_handler(self, signum, arg): # pylint: disable=unused-argument - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received.") - # Kill all container services quickly - # If we're here, we want everything to stop immediately - # and don't care about a gracefully shutdown - self._stop(True) - sys.exit(1) + def _register_exits(self): + signal.signal(signal.SIGINT, self._exit_handler) + signal.signal(signal.SIGTERM, self._exit_handler) + signal.signal(signal.SIGABRT, self._exit_handler) + signal.signal(signal.SIGQUIT, self._exit_handler) - def stop(self, kill=False): - self.test_run.stop(kill) + def _exit_handler(self, signum, arg): # pylint: disable=unused-argument + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received.") + # Kill all container services quickly + # If we're here, we want everything to stop immediately + # and don't care about a gracefully shutdown + self._stop(True) + sys.exit(1) - def start(self): - self.test_run.start() - LOGGER.info("Test Run has finished") + def stop(self, kill=False): + self.test_run.stop(kill) + def start(self): + self.test_run.start() + LOGGER.info("Test Run has finished") def parse_args(argv): - parser = argparse.ArgumentParser(description="Test Run", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-f", "--config-file", default=None, - help="Define the configuration file for Test Run and Network Orchestrator") - parser.add_argument("--no-validate", action="store_true", - help="Turn off the validation of the network after network boot") - parser.add_argument("-net", "--net-only", action="store_true", - help="Run the network only, do not run tests") - parser.add_argument("--single-intf", action="store_true", - help="Single interface mode (experimental)") - args, unknown = parser.parse_known_args() - return args - + parser = argparse.ArgumentParser(description="Test Run", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-f", "--config-file", default=None, + help="Define the configuration file for Test Run and Network Orchestrator") + parser.add_argument("--no-validate", action="store_true", + help="Turn off the validation of the network after network boot") + parser.add_argument("-net", "--net-only", action="store_true", + help="Run the network only, do not run tests") + parser.add_argument("--single-intf", action="store_true", + help="Single interface mode (experimental)") + parsed_args = parser.parse_known_args()[0] + return parsed_args if __name__ == "__main__": - args = parse_args(sys.argv) - runner = TestRunner(config_file=args.config_file, - validate=not args.no_validate, - net_only=args.net_only, - single_intf=args.single_intf) - runner.start() + args = parse_args(sys.argv) + runner = TestRunner(config_file=args.config_file, + validate=not args.no_validate, + net_only=args.net_only, + single_intf=args.single_intf) + runner.start() diff --git a/test_orc/modules/base/python/src/logger.py b/test_orc/modules/base/python/src/logger.py index 641aa16b4..4c8dee9f9 100644 --- a/test_orc/modules/base/python/src/logger.py +++ b/test_orc/modules/base/python/src/logger.py @@ -6,41 +6,34 @@ LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" -_DATE_FORMAT = '%b %02d %H:%M:%S' +_DATE_FORMAT = "%b %02d %H:%M:%S" _DEFAULT_LEVEL = logging.INFO _CONF_DIR = "conf" _CONF_FILE_NAME = "system.json" _LOG_DIR = "/runtime/output/" # Set log level -try: - system_conf_json = json.load( - open(os.path.join(_CONF_DIR, _CONF_FILE_NAME))) - log_level_str = system_conf_json['log_level'] - log_level = logging.getLevelName(log_level_str) -except: - # TODO: Print out warning that log level is incorrect or missing - log_level = _DEFAULT_LEVEL - +system_conf_json = json.load( + open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding="utf-8")) +log_level_str = system_conf_json["log_level"] +log_level = logging.getLevelName(log_level_str) log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) -def add_file_handler(log, logFile): - handler = logging.FileHandler(_LOG_DIR+logFile+".log") - handler.setFormatter(log_format) - log.addHandler(handler) - +def add_file_handler(log, log_file): + handler = logging.FileHandler(_LOG_DIR+log_file+".log") + handler.setFormatter(log_format) + log.addHandler(handler) def add_stream_handler(log): - handler = logging.StreamHandler() - handler.setFormatter(log_format) - log.addHandler(handler) - - -def get_logger(name, logFile=None): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - LOGGERS[name].setLevel(log_level) - add_stream_handler(LOGGERS[name]) - if logFile is not None: - add_file_handler(LOGGERS[name], logFile) - return LOGGERS[name] + handler = logging.StreamHandler() + handler.setFormatter(log_format) + log.addHandler(handler) + +def get_logger(name, log_file=None): + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + LOGGERS[name].setLevel(log_level) + add_stream_handler(LOGGERS[name]) + if log_file is not None: + add_file_handler(LOGGERS[name], log_file) + return LOGGERS[name] diff --git a/test_orc/modules/base/python/src/util.py b/test_orc/modules/base/python/src/util.py index a2dcfbdb1..e705b1186 100644 --- a/test_orc/modules/base/python/src/util.py +++ b/test_orc/modules/base/python/src/util.py @@ -9,17 +9,19 @@ # succesful in running the command. Failure is indicated # by any return code from the process other than zero. def run_command(cmd, output=True): - success = False - LOGGER = logger.get_logger('util') - process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if process.returncode !=0 and output: - err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) - LOGGER.error("Command Failed: " + cmd) - LOGGER.error("Error: " + err_msg) - else: - success = True - if output: - return stdout.strip().decode('utf-8'), stderr - else: - return success + success = False + LOGGER = logger.get_logger('util') + process = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.returncode !=0 and output: + err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) + LOGGER.error("Command Failed: " + cmd) + LOGGER.error("Error: " + err_msg) + else: + success = True + if output: + return stdout.strip().decode("utf-8"), stderr + else: + return success diff --git a/test_orc/modules/baseline/python/src/baseline_module.py b/test_orc/modules/baseline/python/src/baseline_module.py index 80c04ef48..f1f9afc70 100644 --- a/test_orc/modules/baseline/python/src/baseline_module.py +++ b/test_orc/modules/baseline/python/src/baseline_module.py @@ -6,26 +6,27 @@ LOGGER = None class BaselineModule(TestModule): + """An example testing module.""" - def __init__(self, module): - super().__init__(module_name=module, log_name=LOG_NAME) - global LOGGER - LOGGER = self._get_logger() + def __init__(self, module): + super().__init__(module_name=module, log_name=LOG_NAME) + global LOGGER + LOGGER = self._get_logger() - def _baseline_pass(self): - LOGGER.info( - "Running baseline pass test") - LOGGER.info("Baseline pass test finished") - return True + def _baseline_pass(self): + LOGGER.info( + "Running baseline pass test") + LOGGER.info("Baseline pass test finished") + return True - def _baseline_fail(self): - LOGGER.info( - "Running baseline pass test") - LOGGER.info("Baseline pass test finished") - return False + def _baseline_fail(self): + LOGGER.info( + "Running baseline pass test") + LOGGER.info("Baseline pass test finished") + return False - def _baseline_skip(self): - LOGGER.info( - "Running baseline pass test") - LOGGER.info("Baseline pass test finished") - return None \ No newline at end of file + def _baseline_skip(self): + LOGGER.info( + "Running baseline pass test") + LOGGER.info("Baseline pass test finished") + return None diff --git a/test_orc/modules/baseline/python/src/run.py b/test_orc/modules/baseline/python/src/run.py index 8b55484ae..ba070e7c1 100644 --- a/test_orc/modules/baseline/python/src/run.py +++ b/test_orc/modules/baseline/python/src/run.py @@ -11,39 +11,42 @@ RUNTIME = 1500 class BaselineModuleRunner: + """An example runner class for test modules.""" - def __init__(self,module): + def __init__(self,module): - signal.signal(signal.SIGINT, self._handler) - signal.signal(signal.SIGTERM, self._handler) - signal.signal(signal.SIGABRT, self._handler) - signal.signal(signal.SIGQUIT, self._handler) + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) - LOGGER.info("Starting Baseline Module") + LOGGER.info("Starting Baseline Module") - self._test_module = BaselineModule(module) - self._test_module.run_tests() + self._test_module = BaselineModule(module) + self._test_module.run_tests() - def _handler(self, signum, *other): - LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received. Stopping test module...") - LOGGER.info("Test module stopped") - sys.exit(1) + def _handler(self, signum, *other): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping test module...") + LOGGER.info("Test module stopped") + sys.exit(1) def run(argv): - parser = argparse.ArgumentParser(description="Baseline Module Help", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description="Baseline Module Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - "-m", "--module", help="Define the module name to be used to create the log file") + parser.add_argument( + "-m", + "--module", + help="Define the module name to be used to create the log file") - args = parser.parse_args() + args = parser.parse_args() - # For some reason passing in the args from bash adds an extra - # space before the argument so we'll just strip out extra space - BaselineModuleRunner(args.module.strip()) + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + BaselineModuleRunner(args.module.strip()) if __name__ == "__main__": - run(sys.argv) + run(sys.argv) From 10479512662153ef78fd85ac31a03e8fa0c235c5 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 25 May 2023 11:57:29 +0100 Subject: [PATCH 03/12] More formatting fixes --- framework/logger.py | 2 +- framework/testrun.py | 10 +- .../modules/base/python/src/test_module.py | 187 +++++++++--------- test_orc/modules/nmap/python/src/run.py | 51 ++--- 4 files changed, 127 insertions(+), 123 deletions(-) diff --git a/framework/logger.py b/framework/logger.py index 2442aea58..d4702cb38 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -13,7 +13,7 @@ _LOG_DIR = 'runtime/testing/' # Set log level -with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), +with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding='utf-8') as system_conf_file: system_conf_json = json.load(system_conf_file) log_level_str = system_conf_json['log_level'] diff --git a/framework/testrun.py b/framework/testrun.py index d5c70a9ca..94ad2ef9f 100644 --- a/framework/testrun.py +++ b/framework/testrun.py @@ -52,7 +52,7 @@ class TestRun: # pylint: disable=too-few-public-methods orchestrator and user interface. """ - def __init__(self, + def __init__(self, config_file=CONFIG_FILE, validate=True, net_only=False, @@ -97,7 +97,7 @@ def start(self): LOGGER.info('Waiting for devices on the network...') - # Check timeout and whether testing is currently + # Check timeout and whether testing is currently # in progress before stopping time.sleep(RUNTIME) @@ -138,7 +138,7 @@ def _run_tests(self, device): # To Do: Make this configurable time.sleep(60) # Let device bootup - self._test_orc._run_test_modules(device) + self._test_orc.run_test_modules(device) def _stop_network(self, kill=False): self._net_orc.stop(kill=kill) @@ -165,9 +165,9 @@ def _load_devices(self, device_dir): mac_addr = device_config_json.get(DEVICE_MAC_ADDR) test_modules = device_config_json.get(DEVICE_TEST_MODULES) - device = Device(make=device_make, + device = Device(make=device_make, model=device_model, - mac_addr=mac_addr, + mac_addr=mac_addr, test_modules=json.dumps(test_modules)) self._devices.append(device) diff --git a/test_orc/modules/base/python/src/test_module.py b/test_orc/modules/base/python/src/test_module.py index 22b9e0773..89ac8f35a 100644 --- a/test_orc/modules/base/python/src/test_module.py +++ b/test_orc/modules/base/python/src/test_module.py @@ -5,109 +5,110 @@ from datetime import datetime LOGGER = None -RESULTS_DIR = "/runtime/output/" -CONF_FILE = "/testrun/conf/module_config.json" +RESULTS_DIR = '/runtime/output/' +CONF_FILE = '/testrun/conf/module_config.json' class TestModule: + """An example test module.""" - def __init__(self, module_name, log_name): - self._module_name = module_name - self._device_mac = os.environ['DEVICE_MAC'] - self._ipv4_subnet = os.environ['IPV4_SUBNET'] - self._ipv6_subnet = os.environ['IPV6_SUBNET'] - self._add_logger(log_name=log_name, module_name=module_name) - self._config = self._read_config() - self._device_ipv4_addr = None - self._device_ipv6_addr = None + def __init__(self, module_name, log_name): + self._module_name = module_name + self._device_mac = os.environ['DEVICE_MAC'] + self._ipv4_subnet = os.environ['IPV4_SUBNET'] + self._ipv6_subnet = os.environ['IPV6_SUBNET'] + self._add_logger(log_name=log_name, module_name=module_name) + self._config = self._read_config() + self._device_ipv4_addr = None + self._device_ipv6_addr = None - def _add_logger(self, log_name, module_name): - global LOGGER - LOGGER = logger.get_logger(log_name, module_name) + def _add_logger(self, log_name, module_name): + global LOGGER + LOGGER = logger.get_logger(log_name, module_name) - def _get_logger(self): - return LOGGER + def _get_logger(self): + return LOGGER - def _get_tests(self): - device_test_module = self._get_device_test_module() - return self._get_device_tests(device_test_module) + def _get_tests(self): + device_test_module = self._get_device_test_module() + return self._get_device_tests(device_test_module) - def _get_device_tests(self, device_test_module): - module_tests = self._config["config"]["tests"] - if device_test_module is None: - return module_tests - elif not device_test_module["enabled"]: - return [] - else: - for test in module_tests: - # Resolve device specific configurations for the test if it exists - # and update module test config with device config options - if test["name"] in device_test_module["tests"]: - dev_test_config = device_test_module["tests"][test["name"]] - if "config" in test: - test["config"].update(dev_test_config) - return module_tests + def _get_device_tests(self, device_test_module): + module_tests = self._config['config']['tests'] + if device_test_module is None: + return module_tests + elif not device_test_module['enabled']: + return [] + else: + for test in module_tests: + # Resolve device specific configurations for the test if it exists + # and update module test config with device config options + if test['name'] in device_test_module['tests']: + dev_test_config = device_test_module['tests'][test['name']] + if 'config' in test: + test['config'].update(dev_test_config) + return module_tests - def _get_device_test_module(self): - # TODO: Make DEVICE_TEST_MODULES a static string - if 'DEVICE_TEST_MODULES' in os.environ: - test_modules = json.loads(os.environ['DEVICE_TEST_MODULES']) - if self._module_name in test_modules: - return test_modules[self._module_name] - return None + def _get_device_test_module(self): + # TODO: Make DEVICE_TEST_MODULES a static string + if 'DEVICE_TEST_MODULES' in os.environ: + test_modules = json.loads(os.environ['DEVICE_TEST_MODULES']) + if self._module_name in test_modules: + return test_modules[self._module_name] + return None - def run_tests(self): - if self._config["config"]["network"]: - self._device_ipv4_addr = self._get_device_ipv4() - LOGGER.info("Device IP Resolved: " + str(self._device_ipv4_addr)) - tests = self._get_tests() - for test in tests: - test_method_name = "_" + test["name"].replace(".", "_") - result = None - if ("enabled" in test and test["enabled"]) or "enabled" not in test: - LOGGER.info("Attempting to run test: " + test["name"]) - test['start'] = datetime.now().isoformat() - # Resolve the correct python method by test name and run test - if hasattr(self, test_method_name): - if "config" in test: - result = getattr(self, test_method_name)( - config=test["config"]) - else: - result = getattr(self, test_method_name)() - else: - LOGGER.info("Test " + test["name"] + - " not resolved. Skipping") - result = None - else: - LOGGER.info("Test " + test["name"] + - " disabled. Skipping") - if result is not None: - test["result"] = "compliant" if result else "non-compliant" - else: - test["result"] = "skipped" - test['end'] = datetime.now().isoformat() - duration = datetime.fromisoformat(test['end']) - datetime.fromisoformat(test['start']) - test['duration'] = str(duration) - json_results = json.dumps({"results": tests}, indent=2) - self._write_results(json_results) + def run_tests(self): + if self._config['config']['network']: + self._device_ipv4_addr = self._get_device_ipv4() + LOGGER.info('Device IP Resolved: ' + str(self._device_ipv4_addr)) + tests = self._get_tests() + for test in tests: + test_method_name = '_' + test['name'].replace('.', '_') + result = None + if ('enabled' in test and test['enabled']) or 'enabled' not in test: + LOGGER.info('Attempting to run test: ' + test['name']) + test['start'] = datetime.now().isoformat() + # Resolve the correct python method by test name and run test + if hasattr(self, test_method_name): + if 'config' in test: + result = getattr(self, test_method_name)( + config=test['config']) + else: + result = getattr(self, test_method_name)() + else: + LOGGER.info('Test ' + test['name'] + + ' not resolved. Skipping') + result = None + else: + LOGGER.info('Test ' + test['name'] + + ' disabled. Skipping') + if result is not None: + test['result'] = 'compliant' if result else 'non-compliant' + else: + test['result'] = 'skipped' + test['end'] = datetime.now().isoformat() + duration = datetime.fromisoformat(test['end']) - datetime.fromisoformat(test['start']) + test['duration'] = str(duration) + json_results = json.dumps({'results': tests}, indent=2) + self._write_results(json_results) - def _read_config(self): - f = open(CONF_FILE, encoding="utf-8") - config = json.load(f) - f.close() - return config + def _read_config(self): + f = open(CONF_FILE, encoding='utf-8') + config = json.load(f) + f.close() + return config - def _write_results(self, results): - results_file = RESULTS_DIR + self._module_name + "-result.json" - LOGGER.info("Writing results to " + results_file) - f = open(results_file, "w", encoding="utf-8") - f.write(results) - f.close() + def _write_results(self, results): + results_file = RESULTS_DIR + self._module_name + '-result.json' + LOGGER.info('Writing results to ' + results_file) + f = open(results_file, 'w', encoding='utf-8') + f.write(results) + f.close() - def _get_device_ipv4(self): - command = '/testrun/bin/get_ipv4_addr {} {}'.format( - self._ipv4_subnet, self._device_mac.upper()) - text, err = util.run_command(command) - if text: - return text.split("\n")[0] - return None + def _get_device_ipv4(self): + command = f"""/testrun/bin/get_ipv4_addr {self._ipv4_subnet} + {self._device_mac.upper()}""" + text = util.run_command(command)[0] + if text: + return text.split('\n')[0] + return None diff --git a/test_orc/modules/nmap/python/src/run.py b/test_orc/modules/nmap/python/src/run.py index 4c8294769..1aed8f0c0 100644 --- a/test_orc/modules/nmap/python/src/run.py +++ b/test_orc/modules/nmap/python/src/run.py @@ -10,39 +10,42 @@ LOGGER = logger.get_logger('test_module') class NmapModuleRunner: + """Run the NMAP module tests.""" - def __init__(self,module): + def __init__(self,module): - signal.signal(signal.SIGINT, self._handler) - signal.signal(signal.SIGTERM, self._handler) - signal.signal(signal.SIGABRT, self._handler) - signal.signal(signal.SIGQUIT, self._handler) + signal.signal(signal.SIGINT, self._handler) + signal.signal(signal.SIGTERM, self._handler) + signal.signal(signal.SIGABRT, self._handler) + signal.signal(signal.SIGQUIT, self._handler) - LOGGER.info("Starting nmap Module") + LOGGER.info("Starting nmap Module") - self._test_module = NmapModule(module) - self._test_module.run_tests() + self._test_module = NmapModule(module) + self._test_module.run_tests() - def _handler(self, signum, *other): - LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.debug("Exit signal received: " + str(signum)) - if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received. Stopping test module...") - LOGGER.info("Test module stopped") - sys.exit(1) + def _handler(self, signum, *other): + LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.debug("Exit signal received: " + str(signum)) + if signum in (2, signal.SIGTERM): + LOGGER.info("Exit signal received. Stopping test module...") + LOGGER.info("Test module stopped") + sys.exit(1) def run(argv): - parser = argparse.ArgumentParser(description="Nmap Module Help", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description="Nmap Module Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - "-m", "--module", help="Define the module name to be used to create the log file") + parser.add_argument( + "-m", + "--module", + help="Define the module name to be used to create the log file") - args = parser.parse_args() + args = parser.parse_args() - # For some reason passing in the args from bash adds an extra - # space before the argument so we'll just strip out extra space - NmapModuleRunner(args.module.strip()) + # For some reason passing in the args from bash adds an extra + # space before the argument so we'll just strip out extra space + NmapModuleRunner(args.module.strip()) if __name__ == "__main__": - run(sys.argv) + run(sys.argv) From 045bbdc83fdfc9ae6d99a4f59f6e4e02891044c7 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 25 May 2023 12:06:40 +0100 Subject: [PATCH 04/12] More formatting fixes --- .../network/modules/ovs/python/src/logger.py | 12 +- .../modules/ovs/python/src/ovs_control.py | 186 +++++++++--------- net_orc/network/modules/ovs/python/src/run.py | 60 +++--- .../network/modules/ovs/python/src/util.py | 30 +-- .../base/python/src/grpc/start_server.py | 32 ++- .../modules/nmap/python/src/nmap_module.py | 12 +- 6 files changed, 166 insertions(+), 166 deletions(-) diff --git a/net_orc/network/modules/ovs/python/src/logger.py b/net_orc/network/modules/ovs/python/src/logger.py index 50dfb4f50..566a5c75e 100644 --- a/net_orc/network/modules/ovs/python/src/logger.py +++ b/net_orc/network/modules/ovs/python/src/logger.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 import logging -import os -import sys LOGGERS = {} _LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" _DATE_FORMAT = '%b %02d %H:%M:%S' # Set level to debug if set as runtime flag -logging.basicConfig(format=_LOG_FORMAT, datefmt=_DATE_FORMAT, level=logging.INFO) +logging.basicConfig(format=_LOG_FORMAT, + datefmt=_DATE_FORMAT, + level=logging.INFO) def get_logger(name): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - return LOGGERS[name] \ No newline at end of file + if name not in LOGGERS: + LOGGERS[name] = logging.getLogger(name) + return LOGGERS[name] diff --git a/net_orc/network/modules/ovs/python/src/ovs_control.py b/net_orc/network/modules/ovs/python/src/ovs_control.py index 6647dc89e..53406cef2 100644 --- a/net_orc/network/modules/ovs/python/src/ovs_control.py +++ b/net_orc/network/modules/ovs/python/src/ovs_control.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 -#import ipaddress import json import logger -#import os import util CONFIG_FILE = "/ovs/conf/system.json" @@ -13,95 +11,95 @@ class OVSControl: - def __init__(self): - self._int_intf = None - self._dev_intf = None - self._load_config() - - def add_bridge(self,bridgeName): - LOGGER.info("Adding OVS Bridge: " + bridgeName) - # Create the bridge using ovs-vsctl commands - # Uses the --may-exist option to prevent failures - # if this bridge already exists by this name it won't fail - # and will not modify the existing bridge - success=util.run_command("ovs-vsctl --may-exist add-br " + bridgeName) - return success - - def add_port(self,port, bridgeName): - LOGGER.info("Adding Port " + port + " to OVS Bridge: " + bridgeName) - # Add a port to the bridge using ovs-vsctl commands - # Uses the --may-exist option to prevent failures - # if this port already exists on the bridge and will not - # modify the existing bridge - success=util.run_command("ovs-vsctl --may-exist add-port " + bridgeName + " " + port) - return success - - def create_net(self): - LOGGER.info("Creating baseline network") - - # Create data plane - self.add_bridge(DEVICE_BRIDGE) - - # Create control plane - self.add_bridge(INTERNET_BRIDGE) - - # Remove IP from internet adapter - self.set_interface_ip(self._int_intf,"0.0.0.0") - - # Add external interfaces to data and control plane - self.add_port(self._dev_intf,DEVICE_BRIDGE) - self.add_port(self._int_intf,INTERNET_BRIDGE) - - # # Set ports up - self.set_bridge_up(DEVICE_BRIDGE) - self.set_bridge_up(INTERNET_BRIDGE) - - def delete_bridge(self,bridgeName): - LOGGER.info("Deleting OVS Bridge: " + bridgeName) - # Delete the bridge using ovs-vsctl commands - # Uses the --if-exists option to prevent failures - # if this bridge does not exists - success=util.run_command("ovs-vsctl --if-exists del-br " + bridgeName) - return success - - def _load_config(self): - LOGGER.info("Loading Configuration: " + CONFIG_FILE) - config_json = json.load(open(CONFIG_FILE, 'r')) - self._int_intf = config_json['internet_intf'] - self._dev_intf = config_json['device_intf'] - LOGGER.info("Configuration Loaded") - LOGGER.info("Internet Interface: " + self._int_intf) - LOGGER.info("Device Interface: " + self._dev_intf) - - def restore_net(self): - LOGGER.info("Restoring Network...") - # Delete data plane - self.delete_bridge(DEVICE_BRIDGE) - - # Delete control plane - self.delete_bridge(INTERNET_BRIDGE) - - LOGGER.info("Network is restored") - - def show_config(self): - LOGGER.info("Show current config of OVS") - success=util.run_command("ovs-vsctl show") - return success - - def set_bridge_up(self,bridgeName): - LOGGER.info("Setting Bridge device to up state: " + bridgeName) - success=util.run_command("ip link set dev " + bridgeName + " up") - return success - - def set_interface_ip(self,interface, ipAddr): - LOGGER.info("Setting interface " + interface + " to " + ipAddr) - # Remove IP from internet adapter - util.run_command("ifconfig " + interface + " 0.0.0.0") - -if __name__ == '__main__': - ovs = OVSControl() - ovs.create_net() - ovs.show_config() - ovs.restore_net() - ovs.show_config() - + def __init__(self): + self._int_intf = None + self._dev_intf = None + self._load_config() + + def add_bridge(self, bridge_name): + LOGGER.info("Adding OVS Bridge: " + bridge_name) + # Create the bridge using ovs-vsctl commands + # Uses the --may-exist option to prevent failures + # if this bridge already exists by this name it won't fail + # and will not modify the existing bridge + success=util.run_command("ovs-vsctl --may-exist add-br " + bridge_name) + return success + + def add_port(self,port, bridge_name): + LOGGER.info("Adding Port " + port + " to OVS Bridge: " + bridge_name) + # Add a port to the bridge using ovs-vsctl commands + # Uses the --may-exist option to prevent failures + # if this port already exists on the bridge and will not + # modify the existing bridge + success=util.run_command(f"""ovs-vsctl --may-exist + add-port {bridge_name} {port}""") + return success + + def create_net(self): + LOGGER.info("Creating baseline network") + + # Create data plane + self.add_bridge(DEVICE_BRIDGE) + + # Create control plane + self.add_bridge(INTERNET_BRIDGE) + + # Remove IP from internet adapter + self.set_interface_ip(self._int_intf,"0.0.0.0") + + # Add external interfaces to data and control plane + self.add_port(self._dev_intf,DEVICE_BRIDGE) + self.add_port(self._int_intf,INTERNET_BRIDGE) + + # # Set ports up + self.set_bridge_up(DEVICE_BRIDGE) + self.set_bridge_up(INTERNET_BRIDGE) + + def delete_bridge(self,bridge_name): + LOGGER.info("Deleting OVS Bridge: " + bridge_name) + # Delete the bridge using ovs-vsctl commands + # Uses the --if-exists option to prevent failures + # if this bridge does not exists + success=util.run_command("ovs-vsctl --if-exists del-br " + bridge_name) + return success + + def _load_config(self): + LOGGER.info("Loading Configuration: " + CONFIG_FILE) + config_json = json.load(open(CONFIG_FILE, "r", encoding="utf-8")) + self._int_intf = config_json["internet_intf"] + self._dev_intf = config_json["device_intf"] + LOGGER.info("Configuration Loaded") + LOGGER.info("Internet Interface: " + self._int_intf) + LOGGER.info("Device Interface: " + self._dev_intf) + + def restore_net(self): + LOGGER.info("Restoring Network...") + # Delete data plane + self.delete_bridge(DEVICE_BRIDGE) + + # Delete control plane + self.delete_bridge(INTERNET_BRIDGE) + + LOGGER.info("Network is restored") + + def show_config(self): + LOGGER.info("Show current config of OVS") + success=util.run_command("ovs-vsctl show") + return success + + def set_bridge_up(self,bridge_name): + LOGGER.info("Setting Bridge device to up state: " + bridge_name) + success=util.run_command("ip link set dev " + bridge_name + " up") + return success + + def set_interface_ip(self,interface, ip_addr): + LOGGER.info("Setting interface " + interface + " to " + ip_addr) + # Remove IP from internet adapter + util.run_command("ifconfig " + interface + " 0.0.0.0") + +if __name__ == "__main__": + ovs = OVSControl() + ovs.create_net() + ovs.show_config() + ovs.restore_net() + ovs.show_config() diff --git a/net_orc/network/modules/ovs/python/src/run.py b/net_orc/network/modules/ovs/python/src/run.py index 4c1474e74..f91c2dfeb 100644 --- a/net_orc/network/modules/ovs/python/src/run.py +++ b/net_orc/network/modules/ovs/python/src/run.py @@ -2,7 +2,8 @@ import logger import signal -import time +import sys +import time from ovs_control import OVSControl @@ -10,44 +11,45 @@ class OVSControlRun: - def __init__(self): + def __init__(self): - signal.signal(signal.SIGINT, self.handler) - signal.signal(signal.SIGTERM, self.handler) - signal.signal(signal.SIGABRT, self.handler) - signal.signal(signal.SIGQUIT, self.handler) + signal.signal(signal.SIGINT, self.handler) + signal.signal(signal.SIGTERM, self.handler) + signal.signal(signal.SIGABRT, self.handler) + signal.signal(signal.SIGQUIT, self.handler) - LOGGER.info("Starting OVS Control") + LOGGER.info("Starting OVS Control") - # Get all components ready - self._ovs_control = OVSControl() + # Get all components ready + self._ovs_control = OVSControl() - self._ovs_control.restore_net() + self._ovs_control.restore_net() - self._ovs_control.create_net() + self._ovs_control.create_net() - self._ovs_control.show_config() + self._ovs_control.show_config() - # Get network ready (via Network orchestrator) - LOGGER.info("Network is ready. Waiting for device information...") + # Get network ready (via Network orchestrator) + LOGGER.info("Network is ready. Waiting for device information...") - #Loop forever until process is stopped - while True: - LOGGER.info("OVS Running") - time.sleep(1000) + #Loop forever until process is stopped + while True: + LOGGER.info("OVS Running") + time.sleep(1000) - # TODO: This time should be configurable (How long to hold before exiting, this could be infinite too) - #time.sleep(300) + # TODO: This time should be configurable (How long to hold before exiting, + # this could be infinite too) + #time.sleep(300) - # Tear down network - #self._ovs_control.shutdown() + # Tear down network + #self._ovs_control.shutdown() - def handler(self, signum, frame): - LOGGER.info("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.info("Exit signal received: " + str(signum)) - if (signum == 2 or signal == signal.SIGTERM): - LOGGER.info("Exit signal received. Restoring network...") - self._ovs_control.shutdown() - exit(1) + def handler(self, signum, frame): + LOGGER.info("SigtermEnum: " + str(signal.SIGTERM)) + LOGGER.info("Exit signal received: " + str(signum)) + if (signum == 2 or signal == signal.SIGTERM): + LOGGER.info("Exit signal received. Restoring network...") + self._ovs_control.shutdown() + sys.exit(1) ovs = OVSControlRun() diff --git a/net_orc/network/modules/ovs/python/src/util.py b/net_orc/network/modules/ovs/python/src/util.py index 8bb0439bc..c9eba39ff 100644 --- a/net_orc/network/modules/ovs/python/src/util.py +++ b/net_orc/network/modules/ovs/python/src/util.py @@ -3,17 +3,19 @@ def run_command(cmd): - success = False - LOGGER = logger.get_logger('util') - process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if process.returncode !=0: - err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) - LOGGER.error("Command Failed: " + cmd) - LOGGER.error("Error: " + err_msg) - else: - succ_msg = "%s. Code: %s" % (stdout.strip().decode('utf-8'), process.returncode) - LOGGER.info("Command Success: " + cmd) - LOGGER.info("Success: " + succ_msg) - success = True - return success \ No newline at end of file + success = False + LOGGER = logger.get_logger('util') + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.returncode !=0: + err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) + LOGGER.error("Command Failed: " + cmd) + LOGGER.error("Error: " + err_msg) + else: + succ_msg = "%s. Code: %s" % (stdout.strip().decode('utf-8'), process.returncode) + LOGGER.info("Command Success: " + cmd) + LOGGER.info("Success: " + succ_msg) + success = True + return success diff --git a/test_orc/modules/base/python/src/grpc/start_server.py b/test_orc/modules/base/python/src/grpc/start_server.py index 9ed31ffcf..2530fbccb 100644 --- a/test_orc/modules/base/python/src/grpc/start_server.py +++ b/test_orc/modules/base/python/src/grpc/start_server.py @@ -3,32 +3,30 @@ import proto.grpc_pb2_grpc as pb2_grpc import proto.grpc_pb2 as pb2 from network_service import NetworkService -import logging import sys import argparse -DEFAULT_PORT = '5001' +DEFAULT_PORT = "5001" def serve(PORT): - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - pb2_grpc.add_NetworkModuleServicer_to_server(NetworkService(), server) - server.add_insecure_port('[::]:' + PORT) - server.start() - server.wait_for_termination() + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + pb2_grpc.add_NetworkModuleServicer_to_server(NetworkService(), server) + server.add_insecure_port("[::]:" + PORT) + server.start() + server.wait_for_termination() def run(argv): - parser = argparse.ArgumentParser(description="GRPC Server for Network Module", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-p", "--port", default=DEFAULT_PORT, - help="Define the default port to run the server on.") + parser = argparse.ArgumentParser(description="GRPC Server for Network Module", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-p", "--port", default=DEFAULT_PORT, + help="Define the default port to run the server on.") - args = parser.parse_args() + args = parser.parse_args() - PORT = args.port - - print("gRPC server starting on port " + PORT) - serve(PORT) + PORT = args.port + print("gRPC server starting on port " + PORT) + serve(PORT) if __name__ == "__main__": - run(sys.argv) \ No newline at end of file + run(sys.argv) diff --git a/test_orc/modules/nmap/python/src/nmap_module.py b/test_orc/modules/nmap/python/src/nmap_module.py index 477dbe3c6..121df6a62 100644 --- a/test_orc/modules/nmap/python/src/nmap_module.py +++ b/test_orc/modules/nmap/python/src/nmap_module.py @@ -101,13 +101,13 @@ def _check_scan_results(self, test_config): LOGGER.info("Port is closed") result = True else: - LOGGER.info("Port not detected, closed") - result = True + LOGGER.info("Port not detected, closed") + result = True if result is not None: - port_config[port]["result"] = "compliant" if result else "non-compliant" + port_config[port]["result"] = "compliant" if result else "non-compliant" else: - port_config[port]["result"] = "skipped" + port_config[port]["result"] = "skipped" def _scan_scripts(self, tests): scan_results = {} @@ -142,8 +142,8 @@ def _scan_tcp_with_script(self, script_name, ports=None): port_options += " -p" + ports + " " results_file = f"/runtime/output/{self._module_name}-script_name.log" nmap_options = scan_options + port_options + " -oG " + results_file - nmap_results, err = util.run_command( - "nmap " + nmap_options + " " + self._device_ipv4_addr) + nmap_results = util.run_command( + "nmap " + nmap_options + " " + self._device_ipv4_addr)[0] LOGGER.info("Nmap TCP script scan complete") LOGGER.info("nmap script results\n" + str(nmap_results)) return self._process_nmap_results(nmap_results=nmap_results) From 7c43af5868b900f540edd8f56a64e452f35043c2 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 25 May 2023 10:50:06 -0600 Subject: [PATCH 05/12] Misc pylint fixes Fix test module logger --- .../base/python/src/grpc/start_server.py | 12 +++- test_orc/modules/base/python/src/logger.py | 33 ++++++---- .../modules/base/python/src/test_module.py | 12 ++-- test_orc/modules/base/python/src/util.py | 7 +- .../baseline/python/src/baseline_module.py | 10 ++- test_orc/modules/baseline/python/src/run.py | 10 ++- test_orc/modules/dns/python/src/dns_module.py | 23 ++++--- test_orc/modules/dns/python/src/run.py | 18 +++-- .../modules/nmap/python/src/nmap_module.py | 66 +++++++++---------- test_orc/modules/nmap/python/src/run.py | 12 ++-- test_orc/python/src/module.py | 5 +- test_orc/python/src/runner.py | 1 + test_orc/python/src/test_orchestrator.py | 32 ++++----- 13 files changed, 130 insertions(+), 111 deletions(-) diff --git a/test_orc/modules/base/python/src/grpc/start_server.py b/test_orc/modules/base/python/src/grpc/start_server.py index 2530fbccb..970da67fc 100644 --- a/test_orc/modules/base/python/src/grpc/start_server.py +++ b/test_orc/modules/base/python/src/grpc/start_server.py @@ -8,6 +8,7 @@ DEFAULT_PORT = "5001" + def serve(PORT): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) pb2_grpc.add_NetworkModuleServicer_to_server(NetworkService(), server) @@ -15,10 +16,14 @@ def serve(PORT): server.start() server.wait_for_termination() + def run(argv): - parser = argparse.ArgumentParser(description="GRPC Server for Network Module", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-p", "--port", default=DEFAULT_PORT, + parser = argparse.ArgumentParser( + description="GRPC Server for Network Module", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-p", + "--port", + default=DEFAULT_PORT, help="Define the default port to run the server on.") args = parser.parse_args() @@ -28,5 +33,6 @@ def run(argv): print("gRPC server starting on port " + PORT) serve(PORT) + if __name__ == "__main__": run(sys.argv) diff --git a/test_orc/modules/base/python/src/logger.py b/test_orc/modules/base/python/src/logger.py index 4c8dee9f9..42124beea 100644 --- a/test_orc/modules/base/python/src/logger.py +++ b/test_orc/modules/base/python/src/logger.py @@ -1,34 +1,43 @@ -#!/usr/bin/env python3 - +"""Sets up the logger to be used for the test modules.""" import json import logging import os LOGGERS = {} -_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" -_DATE_FORMAT = "%b %02d %H:%M:%S" +_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' +_DATE_FORMAT = '%b %02d %H:%M:%S' _DEFAULT_LEVEL = logging.INFO -_CONF_DIR = "conf" -_CONF_FILE_NAME = "system.json" -_LOG_DIR = "/runtime/output/" +_CONF_DIR = 'conf' +_CONF_FILE_NAME = 'system.json' +_LOG_DIR = '/runtime/output/' # Set log level -system_conf_json = json.load( - open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), encoding="utf-8")) -log_level_str = system_conf_json["log_level"] -log_level = logging.getLevelName(log_level_str) +try: + with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), + encoding='UTF-8') as config_json_file: + system_conf_json = json.load(config_json_file) + + log_level_str = system_conf_json['log_level'] + log_level = logging.getLevelName(log_level_str) +except OSError: + # TODO: Print out warning that log level is incorrect or missing + log_level = _DEFAULT_LEVEL + log_format = logging.Formatter(fmt=_LOG_FORMAT, datefmt=_DATE_FORMAT) + def add_file_handler(log, log_file): - handler = logging.FileHandler(_LOG_DIR+log_file+".log") + handler = logging.FileHandler(_LOG_DIR + log_file + '.log') handler.setFormatter(log_format) log.addHandler(handler) + def add_stream_handler(log): handler = logging.StreamHandler() handler.setFormatter(log_format) log.addHandler(handler) + def get_logger(name, log_file=None): if name not in LOGGERS: LOGGERS[name] = logging.getLogger(name) diff --git a/test_orc/modules/base/python/src/test_module.py b/test_orc/modules/base/python/src/test_module.py index 89ac8f35a..34af4cbb4 100644 --- a/test_orc/modules/base/python/src/test_module.py +++ b/test_orc/modules/base/python/src/test_module.py @@ -71,23 +71,21 @@ def run_tests(self): # Resolve the correct python method by test name and run test if hasattr(self, test_method_name): if 'config' in test: - result = getattr(self, test_method_name)( - config=test['config']) + result = getattr(self, test_method_name)(config=test['config']) else: result = getattr(self, test_method_name)() else: - LOGGER.info('Test ' + test['name'] + - ' not resolved. Skipping') + LOGGER.info('Test ' + test['name'] + ' not resolved. Skipping') result = None else: - LOGGER.info('Test ' + test['name'] + - ' disabled. Skipping') + LOGGER.info('Test ' + test['name'] + ' disabled. Skipping') if result is not None: test['result'] = 'compliant' if result else 'non-compliant' else: test['result'] = 'skipped' test['end'] = datetime.now().isoformat() - duration = datetime.fromisoformat(test['end']) - datetime.fromisoformat(test['start']) + duration = datetime.fromisoformat(test['end']) - datetime.fromisoformat( + test['start']) test['duration'] = str(duration) json_results = json.dumps({'results': tests}, indent=2) self._write_results(json_results) diff --git a/test_orc/modules/base/python/src/util.py b/test_orc/modules/base/python/src/util.py index e705b1186..557f450a6 100644 --- a/test_orc/modules/base/python/src/util.py +++ b/test_orc/modules/base/python/src/util.py @@ -2,6 +2,7 @@ import shlex import logger + # Runs a process at the os level # By default, returns the standard output and error output # If the caller sets optional output parameter to False, @@ -11,11 +12,11 @@ def run_command(cmd, output=True): success = False LOGGER = logger.get_logger('util') - process = subprocess.Popen(shlex.split(cmd), - stdout=subprocess.PIPE, + process = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() - if process.returncode !=0 and output: + if process.returncode != 0 and output: err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) LOGGER.error("Command Failed: " + cmd) LOGGER.error("Error: " + err_msg) diff --git a/test_orc/modules/baseline/python/src/baseline_module.py b/test_orc/modules/baseline/python/src/baseline_module.py index f1f9afc70..9816bd28a 100644 --- a/test_orc/modules/baseline/python/src/baseline_module.py +++ b/test_orc/modules/baseline/python/src/baseline_module.py @@ -5,6 +5,7 @@ LOG_NAME = "test_baseline" LOGGER = None + class BaselineModule(TestModule): """An example testing module.""" @@ -14,19 +15,16 @@ def __init__(self, module): LOGGER = self._get_logger() def _baseline_pass(self): - LOGGER.info( - "Running baseline pass test") + LOGGER.info("Running baseline pass test") LOGGER.info("Baseline pass test finished") return True def _baseline_fail(self): - LOGGER.info( - "Running baseline pass test") + LOGGER.info("Running baseline pass test") LOGGER.info("Baseline pass test finished") return False def _baseline_skip(self): - LOGGER.info( - "Running baseline pass test") + LOGGER.info("Running baseline pass test") LOGGER.info("Baseline pass test finished") return None diff --git a/test_orc/modules/baseline/python/src/run.py b/test_orc/modules/baseline/python/src/run.py index ba070e7c1..89b3a08e4 100644 --- a/test_orc/modules/baseline/python/src/run.py +++ b/test_orc/modules/baseline/python/src/run.py @@ -10,10 +10,11 @@ LOGGER = logger.get_logger('test_module') RUNTIME = 1500 + class BaselineModuleRunner: """An example runner class for test modules.""" - def __init__(self,module): + def __init__(self, module): signal.signal(signal.SIGINT, self._handler) signal.signal(signal.SIGTERM, self._handler) @@ -33,9 +34,11 @@ def _handler(self, signum, *other): LOGGER.info("Test module stopped") sys.exit(1) + def run(argv): - parser = argparse.ArgumentParser(description="Baseline Module Help", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser( + description="Baseline Module Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( "-m", @@ -48,5 +51,6 @@ def run(argv): # space before the argument so we'll just strip out extra space BaselineModuleRunner(args.module.strip()) + if __name__ == "__main__": run(sys.argv) diff --git a/test_orc/modules/dns/python/src/dns_module.py b/test_orc/modules/dns/python/src/dns_module.py index 49c2b5fe9..b161805a5 100644 --- a/test_orc/modules/dns/python/src/dns_module.py +++ b/test_orc/modules/dns/python/src/dns_module.py @@ -7,6 +7,7 @@ CAPTURE_FILE = "/runtime/network/dns.pcap" LOGGER = None + class DNSModule(TestModule): def __init__(self, module): @@ -24,8 +25,8 @@ def _check_dns_traffic(self, tcpdump_filter): return dns_traffic_detected def _dns_network_from_dhcp(self): - LOGGER.info( - "Checking DNS traffic for configured DHCP DNS server: " + self._dns_server) + LOGGER.info("Checking DNS traffic for configured DHCP DNS server: " + + self._dns_server) # Check if the device DNS traffic is to appropriate server tcpdump_filter = "dst port 53 and dst host {} and ether src {}".format( @@ -33,16 +34,15 @@ def _dns_network_from_dhcp(self): result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) - LOGGER.info( - "DNS traffic detected to configured DHCP DNS server: " + str(result)) + LOGGER.info("DNS traffic detected to configured DHCP DNS server: " + + str(result)) return result def _dns_network_from_device(self): LOGGER.info("Checking DNS traffic from device: " + self._device_mac) # Check if the device DNS traffic is to appropriate server - tcpdump_filter = "dst port 53 and ether src {}".format( - self._device_mac) + tcpdump_filter = "dst port 53 and ether src {}".format(self._device_mac) result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) @@ -57,16 +57,15 @@ def _exec_tcpdump(self, tcpdump_filter): Returns List of packets matching the filter """ - command = "tcpdump -tttt -n -r {} {}".format( - CAPTURE_FILE, tcpdump_filter) + command = "tcpdump -tttt -n -r {} {}".format(CAPTURE_FILE, tcpdump_filter) LOGGER.debug("tcpdump command: " + command) process = subprocess.Popen(command, - universal_newlines=True, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + universal_newlines=True, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) text = str(process.stdout.read()).rstrip() LOGGER.debug("tcpdump response: " + text) diff --git a/test_orc/modules/dns/python/src/run.py b/test_orc/modules/dns/python/src/run.py index e09350e6c..06b8aa571 100644 --- a/test_orc/modules/dns/python/src/run.py +++ b/test_orc/modules/dns/python/src/run.py @@ -11,9 +11,10 @@ LOGGER = logger.get_logger(LOG_NAME) RUNTIME = 1500 + class DNSModuleRunner: - def __init__(self,module): + def __init__(self, module): signal.signal(signal.SIGINT, self._handler) signal.signal(signal.SIGTERM, self._handler) @@ -40,20 +41,23 @@ def _handler(self, signum, *other): LOGGER.info("Test module stopped") sys.exit(1) + def run(argv): - parser = argparse.ArgumentParser(description="Test Module DNS", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser( + description="Test Module DNS", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - "-m", - "--module", - help="Define the module name to be used to create the log file") + "-m", + "--module", + help="Define the module name to be used to create the log file") args = parser.parse_args() # For some reason passing in the args from bash adds an extra - # space before the argument so we'll just strip out extra space + # space before the argument so we'll just strip out extra space DNSModuleRunner(args.module.strip()) + if __name__ == "__main__": run(sys.argv) diff --git a/test_orc/modules/nmap/python/src/nmap_module.py b/test_orc/modules/nmap/python/src/nmap_module.py index 121df6a62..cd6ec276b 100644 --- a/test_orc/modules/nmap/python/src/nmap_module.py +++ b/test_orc/modules/nmap/python/src/nmap_module.py @@ -22,8 +22,7 @@ def __init__(self, module): LOGGER = self._get_logger() def _security_nmap_ports(self, config): - LOGGER.info( - "Running security.nmap.ports test") + LOGGER.info("Running security.nmap.ports test") # Delete the enabled key from the config if it exists # to prevent it being treated as a test key @@ -32,12 +31,12 @@ def _security_nmap_ports(self, config): if self._device_ipv4_addr is not None: # Run the monitor method asynchronously to keep this method non-blocking - self._tcp_scan_thread = threading.Thread( - target=self._scan_tcp_ports, args=(config,)) - self._udp_scan_thread = threading.Thread( - target=self._scan_udp_ports, args=(config,)) - self._script_scan_thread = threading.Thread( - target=self._scan_scripts, args=(config,)) + self._tcp_scan_thread = threading.Thread(target=self._scan_tcp_ports, + args=(config, )) + self._udp_scan_thread = threading.Thread(target=self._scan_udp_ports, + args=(config, )) + self._script_scan_thread = threading.Thread(target=self._scan_scripts, + args=(config, )) self._tcp_scan_thread.daemon = True self._udp_scan_thread.daemon = True @@ -47,15 +46,14 @@ def _security_nmap_ports(self, config): self._udp_scan_thread.start() self._script_scan_thread.start() - while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive() or self._script_scan_thread.is_alive(): + while self._tcp_scan_thread.is_alive() or self._udp_scan_thread.is_alive( + ) or self._script_scan_thread.is_alive(): time.sleep(1) LOGGER.debug("TCP scan results: " + str(self._scan_tcp_results)) LOGGER.debug("UDP scan results: " + str(self._scan_udp_results)) - LOGGER.debug("Service scan results: " + - str(self._script_scan_results)) - self._process_port_results( - tests=config) + LOGGER.debug("Service scan results: " + str(self._script_scan_results)) + self._process_port_results(tests=config) LOGGER.info("Unallowed Ports: " + str(self._unallowed_ports)) LOGGER.info("Script scan results:\n" + json.dumps(self._script_scan_results)) @@ -105,7 +103,8 @@ def _check_scan_results(self, test_config): result = True if result is not None: - port_config[port]["result"] = "compliant" if result else "non-compliant" + port_config[port][ + "result"] = "compliant" if result else "non-compliant" else: port_config[port]["result"] = "skipped" @@ -120,16 +119,14 @@ def _scan_scripts(self, tests): if "service_scan" in port_config: LOGGER.info("Service Scan Detected for: " + str(port)) svc = port_config["service_scan"] - scan_results.update( - self._scan_tcp_with_script(svc["script"])) + scan_results.update(self._scan_tcp_with_script(svc["script"])) if "udp_ports" in test_config: for port in test_config["udp_ports"]: if "service_scan" in port: LOGGER.info("Service Scan Detected for: " + str(port)) svc = port["service_scan"] self._scan_udp_with_script(svc["script"], port) - scan_results.update( - self._scan_tcp_with_script(svc["script"])) + scan_results.update(self._scan_tcp_with_script(svc["script"])) self._script_scan_results = scan_results def _scan_tcp_with_script(self, script_name, ports=None): @@ -142,8 +139,8 @@ def _scan_tcp_with_script(self, script_name, ports=None): port_options += " -p" + ports + " " results_file = f"/runtime/output/{self._module_name}-script_name.log" nmap_options = scan_options + port_options + " -oG " + results_file - nmap_results = util.run_command( - "nmap " + nmap_options + " " + self._device_ipv4_addr)[0] + nmap_results = util.run_command("nmap " + nmap_options + " " + + self._device_ipv4_addr)[0] LOGGER.info("Nmap TCP script scan complete") LOGGER.info("nmap script results\n" + str(nmap_results)) return self._process_nmap_results(nmap_results=nmap_results) @@ -157,8 +154,8 @@ def _scan_udp_with_script(self, script_name, ports=None): else: port_options += " -p" + ports + " " nmap_options = scan_options + port_options - nmap_results = util.run_command( - "nmap " + nmap_options + self._device_ipv4_addr)[0] + nmap_results = util.run_command("nmap " + nmap_options + + self._device_ipv4_addr)[0] LOGGER.info("Nmap UDP script scan complete") LOGGER.info("nmap script results\n" + str(nmap_results)) return self._process_nmap_results(nmap_results=nmap_results) @@ -177,12 +174,11 @@ def _scan_tcp_ports(self, tests): ports_to_scan += "," + ",".join(ports) LOGGER.info("Running nmap TCP port scan") LOGGER.info("TCP ports: " + str(ports_to_scan)) - nmap_results = util.run_command( - f"""nmap -sT -sV -Pn -v -p {ports_to_scan} + nmap_results = util.run_command(f"""nmap -sT -sV -Pn -v -p {ports_to_scan} --version-intensity 7 -T4 {self._device_ipv4_addr}""")[0] LOGGER.info("TCP port scan complete") self._scan_tcp_results = self._process_nmap_results( - nmap_results=nmap_results) + nmap_results=nmap_results) def _scan_udp_ports(self, tests): ports = [] @@ -196,7 +192,7 @@ def _scan_udp_ports(self, tests): LOGGER.info("Running nmap UDP port scan") LOGGER.info("UDP ports: " + str(port_list)) nmap_results = util.run_command( - f"nmap -sU -sV -p {port_list} {self._device_ipv4_addr}")[0] + f"nmap -sU -sV -p {port_list} {self._device_ipv4_addr}")[0] LOGGER.info("UDP port scan complete") self._scan_udp_results = self._process_nmap_results( nmap_results=nmap_results) @@ -206,11 +202,10 @@ def _process_nmap_results(self, nmap_results): LOGGER.info("nmap results\n" + str(nmap_results)) if nmap_results: if "Service Info" in nmap_results: - rows = nmap_results.split("PORT")[1].split( - "Service Info")[0].split("\n") + rows = nmap_results.split("PORT")[1].split("Service Info")[0].split( + "\n") elif "PORT" in nmap_results: - rows = nmap_results.split("PORT")[1].split( - "MAC Address")[0].split("\n") + rows = nmap_results.split("PORT")[1].split("MAC Address")[0].split("\n") if rows: for result in rows[1:-1]: # Iterate skipping the header and tail rows cols = result.split() @@ -223,9 +218,12 @@ def _process_nmap_results(self, nmap_results): if len(cols) > 3: # recombine full version information that may contain spaces version = " ".join(cols[3:]) - port_result = {cols[0].split( - "/")[0]: {"state": cols[1], - "service": cols[2], - "version": version}} + port_result = { + cols[0].split("/")[0]: { + "state": cols[1], + "service": cols[2], + "version": version + } + } results.update(port_result) return results diff --git a/test_orc/modules/nmap/python/src/run.py b/test_orc/modules/nmap/python/src/run.py index 1aed8f0c0..4ed1f533c 100644 --- a/test_orc/modules/nmap/python/src/run.py +++ b/test_orc/modules/nmap/python/src/run.py @@ -9,10 +9,11 @@ LOGGER = logger.get_logger('test_module') + class NmapModuleRunner: """Run the NMAP module tests.""" - def __init__(self,module): + def __init__(self, module): signal.signal(signal.SIGINT, self._handler) signal.signal(signal.SIGTERM, self._handler) @@ -32,9 +33,11 @@ def _handler(self, signum, *other): LOGGER.info("Test module stopped") sys.exit(1) + def run(argv): - parser = argparse.ArgumentParser(description="Nmap Module Help", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser( + description="Nmap Module Help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( "-m", @@ -44,8 +47,9 @@ def run(argv): args = parser.parse_args() # For some reason passing in the args from bash adds an extra - # space before the argument so we'll just strip out extra space + # space before the argument so we'll just strip out extra space NmapModuleRunner(args.module.strip()) + if __name__ == "__main__": run(sys.argv) diff --git a/test_orc/python/src/module.py b/test_orc/python/src/module.py index 54f920fa1..72791f86e 100644 --- a/test_orc/python/src/module.py +++ b/test_orc/python/src/module.py @@ -2,8 +2,9 @@ from dataclasses import dataclass from docker.models.containers import Container + @dataclass -class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes +class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Represents a test module.""" name: str = None @@ -13,7 +14,7 @@ class TestModule: # pylint: disable=too-few-public-methods,too-many-instance-att build_file: str = None container: Container = None container_name: str = None - image_name :str = None + image_name: str = None enable_container: bool = True network: bool = True diff --git a/test_orc/python/src/runner.py b/test_orc/python/src/runner.py index cc495bf8d..d82935057 100644 --- a/test_orc/python/src/runner.py +++ b/test_orc/python/src/runner.py @@ -4,6 +4,7 @@ LOGGER = logger.get_logger('runner') + class Runner: """Holds the state of the testing for one device.""" diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 505fe1e49..5cc14ae85 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -50,8 +50,7 @@ def run_test_modules(self, device): for module in self._test_modules: self._run_test_module(module, device) LOGGER.info("All tests complete") - LOGGER.info( - f"""Completed running test modules on device + LOGGER.info(f"""Completed running test modules on device with mac addr {device.mac_addr}""") self._generate_results(device) @@ -64,31 +63,28 @@ def _generate_results(self, device): results["device"]["model"] = device.model results["device"]["mac_addr"] = device.mac_addr for module in self._test_modules: - if module.enable_container and self._is_module_enabled(module,device): + if module.enable_container and self._is_module_enabled(module, device): container_runtime_dir = os.path.join( - self._root_path, - "runtime/test/" + device.mac_addr.replace(":", "") + - "/" + module.name) - results_file = container_runtime_dir+"/"+module.name+"-result.json" + self._root_path, "runtime/test/" + + device.mac_addr.replace(":", "") + "/" + module.name) + results_file = container_runtime_dir + "/" + module.name + "-result.json" try: with open(results_file, "r", encoding="UTF-8") as f: module_results = json.load(f) results[module.name] = module_results - except (FileNotFoundError, - PermissionError, + except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: LOGGER.error("Module Results Errror " + module.name) LOGGER.debug(results_error) out_file = os.path.join( - self._root_path, "runtime/test/" + - device.mac_addr.replace(":", "") + - "/results.json") + self._root_path, + "runtime/test/" + device.mac_addr.replace(":", "") + "/results.json") with open(out_file, "w", encoding="utf-8") as f: - json.dump(results,f,indent=2) + json.dump(results, f, indent=2) return results - def _is_module_enabled(self,module,device): + def _is_module_enabled(self, module, device): enabled = True if device.test_modules is not None: test_modules = json.loads(device.test_modules) @@ -103,7 +99,7 @@ def _run_test_module(self, module, device): if module is None or not module.enable_container: return - if not self._is_module_enabled(module,device): + if not self._is_module_enabled(module, device): return LOGGER.info("Running test module " + module.name) @@ -170,9 +166,9 @@ def _get_module_status(self, module): def _get_test_module(self, name): for test_module in self._test_modules: - if name in [test_module.display_name, - test_module.name, - test_module.dir_name]: + if name in [ + test_module.display_name, test_module.name, test_module.dir_name + ]: return test_module return None From fcedbe767cc32b231239781b5adc69f41ae25d7a Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 25 May 2023 16:41:24 -0600 Subject: [PATCH 06/12] remove unused files --- net_orc/network/modules/ntp/ntp-server.py | 307 ---------------------- net_orc/python/src/logger.py | 31 --- 2 files changed, 338 deletions(-) delete mode 100644 net_orc/network/modules/ntp/ntp-server.py delete mode 100644 net_orc/python/src/logger.py diff --git a/net_orc/network/modules/ntp/ntp-server.py b/net_orc/network/modules/ntp/ntp-server.py deleted file mode 100644 index 9d6a6da8e..000000000 --- a/net_orc/network/modules/ntp/ntp-server.py +++ /dev/null @@ -1,307 +0,0 @@ -import datetime -import socket -import struct -import time -import queue - -import threading -import select - -taskQueue = queue.Queue() -stop_flag = False - -def system_to_ntp_time(timestamp): - """Convert a system time to a NTP time. - - Parameters: - timestamp -- timestamp in system time - - Returns: - corresponding NTP time - """ - return timestamp + NTP.NTP_DELTA - -def _to_int(timestamp): - """Return the integral part of a timestamp. - - Parameters: - timestamp -- NTP timestamp - - Retuns: - integral part - """ - return int(timestamp) - -def _to_frac(timestamp, n=32): - """Return the fractional part of a timestamp. - - Parameters: - timestamp -- NTP timestamp - n -- number of bits of the fractional part - - Retuns: - fractional part - """ - return int(abs(timestamp - _to_int(timestamp)) * 2**n) - -def _to_time(integ, frac, n=32): - """Return a timestamp from an integral and fractional part. - - Parameters: - integ -- integral part - frac -- fractional part - n -- number of bits of the fractional part - - Retuns: - timestamp - """ - return integ + float(frac)/2**n - -class NTPException(Exception): - """Exception raised by this module.""" - pass - -class NTP: - """Helper class defining constants.""" - - _SYSTEM_EPOCH = datetime.date(*time.gmtime(0)[0:3]) - """system epoch""" - _NTP_EPOCH = datetime.date(1900, 1, 1) - """NTP epoch""" - NTP_DELTA = (_SYSTEM_EPOCH - _NTP_EPOCH).days * 24 * 3600 - """delta between system and NTP time""" - - REF_ID_TABLE = { - 'DNC': "DNC routing protocol", - 'NIST': "NIST public modem", - 'TSP': "TSP time protocol", - 'DTS': "Digital Time Service", - 'ATOM': "Atomic clock (calibrated)", - 'VLF': "VLF radio (OMEGA, etc)", - 'callsign': "Generic radio", - 'LORC': "LORAN-C radionavidation", - 'GOES': "GOES UHF environment satellite", - 'GPS': "GPS UHF satellite positioning", - } - """reference identifier table""" - - STRATUM_TABLE = { - 0: "unspecified", - 1: "primary reference", - } - """stratum table""" - - MODE_TABLE = { - 0: "unspecified", - 1: "symmetric active", - 2: "symmetric passive", - 3: "client", - 4: "server", - 5: "broadcast", - 6: "reserved for NTP control messages", - 7: "reserved for private use", - } - """mode table""" - - LEAP_TABLE = { - 0: "no warning", - 1: "last minute has 61 seconds", - 2: "last minute has 59 seconds", - 3: "alarm condition (clock not synchronized)", - } - """leap indicator table""" - -class NTPPacket: - """NTP packet class. - - This represents an NTP packet. - """ - - _PACKET_FORMAT = "!B B B b 11I" - """packet format to pack/unpack""" - - def __init__(self, version=4, mode=3, tx_timestamp=0): - """Constructor. - - Parameters: - version -- NTP version - mode -- packet mode (client, server) - tx_timestamp -- packet transmit timestamp - """ - self.leap = 0 - """leap second indicator""" - self.version = version - """version""" - self.mode = mode - """mode""" - self.stratum = 0 - """stratum""" - self.poll = 0 - """poll interval""" - self.precision = 0 - """precision""" - self.root_delay = 0 - """root delay""" - self.root_dispersion = 0 - """root dispersion""" - self.ref_id = 0 - """reference clock identifier""" - self.ref_timestamp = 0 - """reference timestamp""" - self.orig_timestamp = 0 - self.orig_timestamp_high = 0 - self.orig_timestamp_low = 0 - """originate timestamp""" - self.recv_timestamp = 0 - """receive timestamp""" - self.tx_timestamp = tx_timestamp - self.tx_timestamp_high = 0 - self.tx_timestamp_low = 0 - """tansmit timestamp""" - - def to_data(self): - """Convert this NTPPacket to a buffer that can be sent over a socket. - - Returns: - buffer representing this packet - - Raises: - NTPException -- in case of invalid field - """ - try: - packed = struct.pack(NTPPacket._PACKET_FORMAT, - (self.leap << 6 | self.version << 3 | self.mode), - self.stratum, - self.poll, - self.precision, - _to_int(self.root_delay) << 16 | _to_frac(self.root_delay, 16), - _to_int(self.root_dispersion) << 16 | - _to_frac(self.root_dispersion, 16), - self.ref_id, - _to_int(self.ref_timestamp), - _to_frac(self.ref_timestamp), - #Change by lichen, avoid loss of precision - self.orig_timestamp_high, - self.orig_timestamp_low, - _to_int(self.recv_timestamp), - _to_frac(self.recv_timestamp), - _to_int(self.tx_timestamp), - _to_frac(self.tx_timestamp)) - except struct.error: - raise NTPException("Invalid NTP packet fields.") - return packed - - def from_data(self, data): - """Populate this instance from a NTP packet payload received from - the network. - - Parameters: - data -- buffer payload - - Raises: - NTPException -- in case of invalid packet format - """ - try: - unpacked = struct.unpack(NTPPacket._PACKET_FORMAT, - data[0:struct.calcsize(NTPPacket._PACKET_FORMAT)]) - except struct.error: - raise NTPException("Invalid NTP packet.") - - self.leap = unpacked[0] >> 6 & 0x3 - self.version = unpacked[0] >> 3 & 0x7 - self.mode = unpacked[0] & 0x7 - self.stratum = unpacked[1] - self.poll = unpacked[2] - self.precision = unpacked[3] - self.root_delay = float(unpacked[4])/2**16 - self.root_dispersion = float(unpacked[5])/2**16 - self.ref_id = unpacked[6] - self.ref_timestamp = _to_time(unpacked[7], unpacked[8]) - self.orig_timestamp = _to_time(unpacked[9], unpacked[10]) - self.orig_timestamp_high = unpacked[9] - self.orig_timestamp_low = unpacked[10] - self.recv_timestamp = _to_time(unpacked[11], unpacked[12]) - self.tx_timestamp = _to_time(unpacked[13], unpacked[14]) - self.tx_timestamp_high = unpacked[13] - self.tx_timestamp_low = unpacked[14] - - def GetTxTimeStamp(self): - return (self.tx_timestamp_high,self.tx_timestamp_low) - - def SetOriginTimeStamp(self,high,low): - self.orig_timestamp_high = high - self.orig_timestamp_low = low - -class RecvThread(threading.Thread): - - def __init__(self,socket): - threading.Thread.__init__(self) - self.socket = socket - - def run(self): - global t,stop_flag - while True: - if stop_flag == True: - print("RecvThread Ended") - break - rlist,wlist,elist = select.select([self.socket],[],[],1) - if len(rlist) != 0: - print("Received %d packets" % len(rlist)) - for tempSocket in rlist: - try: - data,addr = tempSocket.recvfrom(1024) - recvTimestamp = recvTimestamp = system_to_ntp_time(time.time()) - taskQueue.put((data,addr,recvTimestamp)) - except socket.error as msg: - print(msg) - -class WorkThread(threading.Thread): - - def __init__(self,socket): - threading.Thread.__init__(self) - self.socket = socket - - def run(self): - global taskQueue,stop_flag - while True: - if stop_flag is True: - print("WorkThread Ended") - break - try: - data,addr,recvTimestamp = taskQueue.get(timeout=1) - recvPacket = NTPPacket() - recvPacket.from_data(data) - timeStamp_high,timeStamp_low = recvPacket.GetTxTimeStamp() - sendPacket = NTPPacket(version=4,mode=4) - sendPacket.stratum = 2 - sendPacket.poll = 10 - sendPacket.ref_timestamp = recvTimestamp-5 - sendPacket.SetOriginTimeStamp(timeStamp_high,timeStamp_low) - sendPacket.recv_timestamp = recvTimestamp - sendPacket.tx_timestamp = system_to_ntp_time(time.time()) - socket.sendto(sendPacket.to_data(),addr) - print("Sent to %s:%d" % (addr[0],addr[1])) - except queue.Empty: - continue - -listen_ip = "0.0.0.0" -listen_port = 123 -socket = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) -socket.bind((listen_ip,listen_port)) -print(f"local socket: {socket.getsockname()}") -recvThread = RecvThread(socket) -recvThread.start() -workThread = WorkThread(socket) -workThread.start() - -while True: - try: - time.sleep(0.5) - except KeyboardInterrupt: - print("Exiting...") - stop_flag = True - recvThread.join() - workThread.join() - #socket.close() - print("Exited") - break diff --git a/net_orc/python/src/logger.py b/net_orc/python/src/logger.py deleted file mode 100644 index aaf690c8a..000000000 --- a/net_orc/python/src/logger.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Sets up the logger to be used for the network orchestrator.""" -import json -import logging -import os - -LOGGERS = {} -_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' -_DATE_FORMAT = '%b %02d %H:%M:%S' -_DEFAULT_LEVEL = logging.INFO -_CONF_DIR = 'conf' -_CONF_FILE_NAME = 'system.json' - -# Set log level -try: - - with open(os.path.join(_CONF_DIR, _CONF_FILE_NAME), - encoding='UTF-8') as config_json_file: - system_conf_json = json.load(config_json_file) - - log_level_str = system_conf_json['log_level'] - LOG_LEVEL = logging.getLevelName(log_level_str) -except OSError: - LOG_LEVEL = _DEFAULT_LEVEL - -logging.basicConfig(format=_LOG_FORMAT, datefmt=_DATE_FORMAT, level=LOG_LEVEL) - - -def get_logger(name): - if name not in LOGGERS: - LOGGERS[name] = logging.getLogger(name) - return LOGGERS[name] From 04264d9af7a7fee8f8d92eeabdd92cd98ddde357 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 25 May 2023 17:14:30 -0600 Subject: [PATCH 07/12] more formatting --- framework/test_runner.py | 45 ++++++++++++------- net_orc/python/src/network_orchestrator.py | 2 +- net_orc/python/src/network_validator.py | 2 +- .../base/python/src/grpc/start_server.py | 29 ++++++------ .../modules/base/python/src/test_module.py | 15 +++---- test_orc/modules/base/python/src/util.py | 11 ++--- testing/test_baseline.py | 4 +- 7 files changed, 61 insertions(+), 47 deletions(-) diff --git a/framework/test_runner.py b/framework/test_runner.py index 95f3e4208..0733d4353 100644 --- a/framework/test_runner.py +++ b/framework/test_runner.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - """Wrapper for the TestRun that simplifies virtual testing procedure by allowing direct calling from the command line. @@ -16,11 +15,15 @@ LOGGER = logger.get_logger("runner") + class TestRunner: """Controls and starts the Test Run application.""" - def __init__(self, config_file=None, validate=True, - net_only=False, single_intf=False): + def __init__(self, + config_file=None, + validate=True, + net_only=False, + single_intf=False): self._register_exits() self.test_run = TestRun(config_file=config_file, validate=validate, @@ -50,22 +53,34 @@ def start(self): self.test_run.start() LOGGER.info("Test Run has finished") -def parse_args(argv): - parser = argparse.ArgumentParser(description="Test Run", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-f", "--config-file", default=None, - help="Define the configuration file for Test Run and Network Orchestrator") - parser.add_argument("--no-validate", action="store_true", - help="Turn off the validation of the network after network boot") - parser.add_argument("-net", "--net-only", action="store_true", - help="Run the network only, do not run tests") - parser.add_argument("--single-intf", action="store_true", - help="Single interface mode (experimental)") + +def parse_args(): + parser = argparse.ArgumentParser( + description="Test Run", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "-f", + "--config-file", + default=None, + help="Define the configuration file for Test Run and Network Orchestrator" + ) + parser.add_argument( + "--no-validate", + action="store_true", + help="Turn off the validation of the network after network boot") + parser.add_argument("-net", + "--net-only", + action="store_true", + help="Run the network only, do not run tests") + parser.add_argument("--single-intf", + action="store_true", + help="Single interface mode (experimental)") parsed_args = parser.parse_known_args()[0] return parsed_args + if __name__ == "__main__": - args = parse_args(sys.argv) + args = parse_args() runner = TestRunner(config_file=args.config_file, validate=not args.no_validate, net_only=args.net_only, diff --git a/net_orc/python/src/network_orchestrator.py b/net_orc/python/src/network_orchestrator.py index 53a94b795..0c5c6c4f5 100644 --- a/net_orc/python/src/network_orchestrator.py +++ b/net_orc/python/src/network_orchestrator.py @@ -13,7 +13,7 @@ import docker from docker.types import Mount import logger -import util +from . import util from listener import Listener from network_device import NetworkDevice from network_event import NetworkEvent diff --git a/net_orc/python/src/network_validator.py b/net_orc/python/src/network_validator.py index 83ca6f671..00cb99911 100644 --- a/net_orc/python/src/network_validator.py +++ b/net_orc/python/src/network_validator.py @@ -7,7 +7,7 @@ from docker.types import Mount import getpass import logger -import util +from . import util LOGGER = logger.get_logger('validator') OUTPUT_DIR = 'runtime/validation' diff --git a/test_orc/modules/base/python/src/grpc/start_server.py b/test_orc/modules/base/python/src/grpc/start_server.py index 970da67fc..b4016c831 100644 --- a/test_orc/modules/base/python/src/grpc/start_server.py +++ b/test_orc/modules/base/python/src/grpc/start_server.py @@ -1,38 +1,37 @@ +"""Base class for starting the gRPC server for a network module.""" from concurrent import futures import grpc import proto.grpc_pb2_grpc as pb2_grpc -import proto.grpc_pb2 as pb2 from network_service import NetworkService -import sys import argparse -DEFAULT_PORT = "5001" +DEFAULT_PORT = '5001' -def serve(PORT): +def serve(port): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) pb2_grpc.add_NetworkModuleServicer_to_server(NetworkService(), server) - server.add_insecure_port("[::]:" + PORT) + server.add_insecure_port('[::]:' + port) server.start() server.wait_for_termination() -def run(argv): +def run(): parser = argparse.ArgumentParser( - description="GRPC Server for Network Module", + description='GRPC Server for Network Module', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-p", - "--port", + parser.add_argument('-p', + '--port', default=DEFAULT_PORT, - help="Define the default port to run the server on.") + help='Define the default port to run the server on.') args = parser.parse_args() - PORT = args.port + port = args.port - print("gRPC server starting on port " + PORT) - serve(PORT) + print('gRPC server starting on port ' + port) + serve(port) -if __name__ == "__main__": - run(sys.argv) +if __name__ == '__main__': + run() diff --git a/test_orc/modules/base/python/src/test_module.py b/test_orc/modules/base/python/src/test_module.py index 34af4cbb4..727ff5c71 100644 --- a/test_orc/modules/base/python/src/test_module.py +++ b/test_orc/modules/base/python/src/test_module.py @@ -1,7 +1,8 @@ +"""Base class for all core test module functions""" import json import logger import os -import util +from . import util from datetime import datetime LOGGER = None @@ -91,20 +92,18 @@ def run_tests(self): self._write_results(json_results) def _read_config(self): - f = open(CONF_FILE, encoding='utf-8') - config = json.load(f) - f.close() + with open(CONF_FILE, encoding='utf-8') as f: + config = json.load(f) return config def _write_results(self, results): results_file = RESULTS_DIR + self._module_name + '-result.json' LOGGER.info('Writing results to ' + results_file) - f = open(results_file, 'w', encoding='utf-8') - f.write(results) - f.close() + with open(results_file, 'w', encoding='utf-8') as f: + f.write(results) def _get_device_ipv4(self): - command = f"""/testrun/bin/get_ipv4_addr {self._ipv4_subnet} + command = f"""/testrun/bin/get_ipv4_addr {self._ipv4_subnet} {self._device_mac.upper()}""" text = util.run_command(command)[0] if text: diff --git a/test_orc/modules/base/python/src/util.py b/test_orc/modules/base/python/src/util.py index 557f450a6..d387db796 100644 --- a/test_orc/modules/base/python/src/util.py +++ b/test_orc/modules/base/python/src/util.py @@ -1,7 +1,9 @@ +"""Provides basic utilities for a test module.""" import subprocess import shlex import logger +LOGGER = logger.get_logger('util') # Runs a process at the os level # By default, returns the standard output and error output @@ -11,18 +13,17 @@ # by any return code from the process other than zero. def run_command(cmd, output=True): success = False - LOGGER = logger.get_logger('util') process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode != 0 and output: - err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) - LOGGER.error("Command Failed: " + cmd) - LOGGER.error("Error: " + err_msg) + err_msg = f'{stderr.strip()}. Code: {process.returncode}' + LOGGER.error('Command Failed: ' + cmd) + LOGGER.error('Error: ' + err_msg) else: success = True if output: - return stdout.strip().decode("utf-8"), stderr + return stdout.strip().decode('utf-8'), stderr else: return success diff --git a/testing/test_baseline.py b/testing/test_baseline.py index 6f6240c27..b356983dd 100644 --- a/testing/test_baseline.py +++ b/testing/test_baseline.py @@ -23,7 +23,7 @@ def validator_results(): encoding='utf-8') as f: return json.load(f) -@pytest.mark.skip(reason="requires internet") +@pytest.mark.skip(reason='requires internet') def test_internet_connectivity(container_data): assert container_data['network']['internet'] == 200 @@ -47,7 +47,7 @@ def test_dns_server_resolves(container_data): assert re.match(r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}', container_data['dns_response']) -@pytest.mark.skip(reason="requires internet") +@pytest.mark.skip(reason='requires internet') def test_validator_results_compliant(validator_results): results = [True if x['result'] == 'compliant' else False for x in validator_results['results']] From 15a9b34dde4b4e16451251fe1852ed7b94267bad Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 26 May 2023 08:15:01 -0600 Subject: [PATCH 08/12] revert breaking pylint changes --- net_orc/python/src/network_orchestrator.py | 2 +- net_orc/python/src/network_validator.py | 2 +- test_orc/modules/base/python/src/test_module.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/net_orc/python/src/network_orchestrator.py b/net_orc/python/src/network_orchestrator.py index 0c5c6c4f5..53a94b795 100644 --- a/net_orc/python/src/network_orchestrator.py +++ b/net_orc/python/src/network_orchestrator.py @@ -13,7 +13,7 @@ import docker from docker.types import Mount import logger -from . import util +import util from listener import Listener from network_device import NetworkDevice from network_event import NetworkEvent diff --git a/net_orc/python/src/network_validator.py b/net_orc/python/src/network_validator.py index 00cb99911..83ca6f671 100644 --- a/net_orc/python/src/network_validator.py +++ b/net_orc/python/src/network_validator.py @@ -7,7 +7,7 @@ from docker.types import Mount import getpass import logger -from . import util +import util LOGGER = logger.get_logger('validator') OUTPUT_DIR = 'runtime/validation' diff --git a/test_orc/modules/base/python/src/test_module.py b/test_orc/modules/base/python/src/test_module.py index 727ff5c71..8e10a3637 100644 --- a/test_orc/modules/base/python/src/test_module.py +++ b/test_orc/modules/base/python/src/test_module.py @@ -2,7 +2,7 @@ import json import logger import os -from . import util +import util from datetime import datetime LOGGER = None From b86e21c78edbce43cb14be059b450902d6fd4590 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 26 May 2023 10:24:35 -0600 Subject: [PATCH 09/12] more formatting --- .../baseline/python/src/baseline_module.py | 4 +-- test_orc/modules/baseline/python/src/run.py | 29 +++++++++---------- .../modules/nmap/python/src/nmap_module.py | 18 +++++------- test_orc/modules/nmap/python/src/run.py | 29 +++++++++---------- test_orc/python/src/test_orchestrator.py | 2 +- 5 files changed, 38 insertions(+), 44 deletions(-) diff --git a/test_orc/modules/baseline/python/src/baseline_module.py b/test_orc/modules/baseline/python/src/baseline_module.py index 9816bd28a..083123436 100644 --- a/test_orc/modules/baseline/python/src/baseline_module.py +++ b/test_orc/modules/baseline/python/src/baseline_module.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""Baseline test module""" from test_module import TestModule LOG_NAME = "test_baseline" @@ -27,4 +26,3 @@ def _baseline_fail(self): def _baseline_skip(self): LOGGER.info("Running baseline pass test") LOGGER.info("Baseline pass test finished") - return None diff --git a/test_orc/modules/baseline/python/src/run.py b/test_orc/modules/baseline/python/src/run.py index 89b3a08e4..1892ed8ae 100644 --- a/test_orc/modules/baseline/python/src/run.py +++ b/test_orc/modules/baseline/python/src/run.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""Run Baseline module""" import argparse import signal import sys @@ -21,29 +20,29 @@ def __init__(self, module): signal.signal(signal.SIGABRT, self._handler) signal.signal(signal.SIGQUIT, self._handler) - LOGGER.info("Starting Baseline Module") + LOGGER.info('Starting Baseline Module') self._test_module = BaselineModule(module) self._test_module.run_tests() - def _handler(self, signum, *other): - LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.debug("Exit signal received: " + str(signum)) + def _handler(self, signum): + LOGGER.debug('SigtermEnum: ' + str(signal.SIGTERM)) + LOGGER.debug('Exit signal received: ' + str(signum)) if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received. Stopping test module...") - LOGGER.info("Test module stopped") + LOGGER.info('Exit signal received. Stopping test module...') + LOGGER.info('Test module stopped') sys.exit(1) -def run(argv): +def run(): parser = argparse.ArgumentParser( - description="Baseline Module Help", + description='Baseline Module Help', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - "-m", - "--module", - help="Define the module name to be used to create the log file") + '-m', + '--module', + help='Define the module name to be used to create the log file') args = parser.parse_args() @@ -52,5 +51,5 @@ def run(argv): BaselineModuleRunner(args.module.strip()) -if __name__ == "__main__": - run(sys.argv) +if __name__ == '__main__': + run() diff --git a/test_orc/modules/nmap/python/src/nmap_module.py b/test_orc/modules/nmap/python/src/nmap_module.py index cd6ec276b..876343a0f 100644 --- a/test_orc/modules/nmap/python/src/nmap_module.py +++ b/test_orc/modules/nmap/python/src/nmap_module.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""NMAP test module""" import time import util import json @@ -11,7 +10,7 @@ class NmapModule(TestModule): - + """NMAP Test module""" def __init__(self, module): super().__init__(module_name=module, log_name=LOG_NAME) self._unallowed_ports = [] @@ -82,13 +81,13 @@ def _check_scan_results(self, test_config): if self._script_scan_results is not None: scan_results.update(self._script_scan_results) if port_config is not None: - for port in port_config: + for port, config in port_config.items(): result = None LOGGER.info("Checking port: " + str(port)) - LOGGER.debug("Port config: " + str(port_config[port])) + LOGGER.debug("Port config: " + str(config)) if port in scan_results: if scan_results[port]["state"] == "open": - if not port_config[port]["allowed"]: + if not config["allowed"]: LOGGER.info("Unallowed port open") self._unallowed_ports.append(str(port)) result = False @@ -103,10 +102,9 @@ def _check_scan_results(self, test_config): result = True if result is not None: - port_config[port][ - "result"] = "compliant" if result else "non-compliant" + config["result"] = "compliant" if result else "non-compliant" else: - port_config[port]["result"] = "skipped" + config["result"] = "skipped" def _scan_scripts(self, tests): scan_results = {} @@ -174,7 +172,7 @@ def _scan_tcp_ports(self, tests): ports_to_scan += "," + ",".join(ports) LOGGER.info("Running nmap TCP port scan") LOGGER.info("TCP ports: " + str(ports_to_scan)) - nmap_results = util.run_command(f"""nmap -sT -sV -Pn -v -p {ports_to_scan} + nmap_results = util.run_command(f"""nmap -sT -sV -Pn -v -p {ports_to_scan} --version-intensity 7 -T4 {self._device_ipv4_addr}""")[0] LOGGER.info("TCP port scan complete") self._scan_tcp_results = self._process_nmap_results( diff --git a/test_orc/modules/nmap/python/src/run.py b/test_orc/modules/nmap/python/src/run.py index 4ed1f533c..78eb601bb 100644 --- a/test_orc/modules/nmap/python/src/run.py +++ b/test_orc/modules/nmap/python/src/run.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""Run NMAP module""" import argparse import signal import sys @@ -20,29 +19,29 @@ def __init__(self, module): signal.signal(signal.SIGABRT, self._handler) signal.signal(signal.SIGQUIT, self._handler) - LOGGER.info("Starting nmap Module") + LOGGER.info('Starting nmap Module') self._test_module = NmapModule(module) self._test_module.run_tests() - def _handler(self, signum, *other): - LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.debug("Exit signal received: " + str(signum)) + def _handler(self, signum): + LOGGER.debug('SigtermEnum: ' + str(signal.SIGTERM)) + LOGGER.debug('Exit signal received: ' + str(signum)) if signum in (2, signal.SIGTERM): - LOGGER.info("Exit signal received. Stopping test module...") - LOGGER.info("Test module stopped") + LOGGER.info('Exit signal received. Stopping test module...') + LOGGER.info('Test module stopped') sys.exit(1) -def run(argv): +def run(): parser = argparse.ArgumentParser( - description="Nmap Module Help", + description='Nmap Module Help', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - "-m", - "--module", - help="Define the module name to be used to create the log file") + '-m', + '--module', + help='Define the module name to be used to create the log file') args = parser.parse_args() @@ -51,5 +50,5 @@ def run(argv): NmapModuleRunner(args.module.strip()) -if __name__ == "__main__": - run(sys.argv) +if __name__ == '__main__': + run() diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index 5cc14ae85..bac6c7241 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -67,7 +67,7 @@ def _generate_results(self, device): container_runtime_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/" + module.name) - results_file = container_runtime_dir + "/" + module.name + "-result.json" + results_file = f"{container_runtime_dir} / {module.name} -results.json" try: with open(results_file, "r", encoding="UTF-8") as f: module_results = json.load(f) From 289f32bb264375a89e4f54862c1307ade4149291 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 26 May 2023 10:45:03 -0600 Subject: [PATCH 10/12] fix results file --- test_orc/python/src/test_orchestrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_orc/python/src/test_orchestrator.py b/test_orc/python/src/test_orchestrator.py index bac6c7241..4b65bae12 100644 --- a/test_orc/python/src/test_orchestrator.py +++ b/test_orc/python/src/test_orchestrator.py @@ -67,7 +67,7 @@ def _generate_results(self, device): container_runtime_dir = os.path.join( self._root_path, "runtime/test/" + device.mac_addr.replace(":", "") + "/" + module.name) - results_file = f"{container_runtime_dir} / {module.name} -results.json" + results_file = f"{container_runtime_dir}/{module.name}-result.json" try: with open(results_file, "r", encoding="UTF-8") as f: module_results = json.load(f) From 5670543339ed7661996d669156196d100c0b6c27 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 26 May 2023 11:03:22 -0600 Subject: [PATCH 11/12] More formatting --- test_orc/modules/dns/python/src/dns_module.py | 36 +++++++++---------- test_orc/modules/dns/python/src/run.py | 11 +++--- test_orc/modules/nmap/python/src/run.py | 2 +- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/test_orc/modules/dns/python/src/dns_module.py b/test_orc/modules/dns/python/src/dns_module.py index b161805a5..58ce48123 100644 --- a/test_orc/modules/dns/python/src/dns_module.py +++ b/test_orc/modules/dns/python/src/dns_module.py @@ -1,52 +1,52 @@ -#!/usr/bin/env python3 - +"""DNS test module""" import subprocess from test_module import TestModule -LOG_NAME = "test_dns" -CAPTURE_FILE = "/runtime/network/dns.pcap" +LOG_NAME = 'test_dns' +CAPTURE_FILE = '/runtime/network/dns.pcap' LOGGER = None class DNSModule(TestModule): + """DNS Test module""" def __init__(self, module): super().__init__(module_name=module, log_name=LOG_NAME) - self._dns_server = "10.10.10.4" + self._dns_server = '10.10.10.4' global LOGGER LOGGER = self._get_logger() def _check_dns_traffic(self, tcpdump_filter): to_dns = self._exec_tcpdump(tcpdump_filter) num_query_dns = len(to_dns) - LOGGER.info("DNS queries found: " + str(num_query_dns)) + LOGGER.info('DNS queries found: ' + str(num_query_dns)) dns_traffic_detected = len(to_dns) > 0 - LOGGER.info("DNS traffic detected: " + str(dns_traffic_detected)) + LOGGER.info('DNS traffic detected: ' + str(dns_traffic_detected)) return dns_traffic_detected def _dns_network_from_dhcp(self): - LOGGER.info("Checking DNS traffic for configured DHCP DNS server: " + + LOGGER.info('Checking DNS traffic for configured DHCP DNS server: ' + self._dns_server) # Check if the device DNS traffic is to appropriate server - tcpdump_filter = "dst port 53 and dst host {} and ether src {}".format( - self._dns_server, self._device_mac) + tcpdump_filter = (f'dst port 53 and dst host {self._dns_server}', + f' and ether src {self._device_mac}') result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) - LOGGER.info("DNS traffic detected to configured DHCP DNS server: " + + LOGGER.info('DNS traffic detected to configured DHCP DNS server: ' + str(result)) return result def _dns_network_from_device(self): - LOGGER.info("Checking DNS traffic from device: " + self._device_mac) + LOGGER.info('Checking DNS traffic from device: ' + self._device_mac) # Check if the device DNS traffic is to appropriate server - tcpdump_filter = "dst port 53 and ether src {}".format(self._device_mac) + tcpdump_filter = f'dst port 53 and ether src {self._device_mac}' result = self._check_dns_traffic(tcpdump_filter=tcpdump_filter) - LOGGER.info("DNS traffic detected from device: " + str(result)) + LOGGER.info('DNS traffic detected from device: ' + str(result)) return result def _exec_tcpdump(self, tcpdump_filter): @@ -57,9 +57,9 @@ def _exec_tcpdump(self, tcpdump_filter): Returns List of packets matching the filter """ - command = "tcpdump -tttt -n -r {} {}".format(CAPTURE_FILE, tcpdump_filter) + command = f'tcpdump -tttt -n -r {CAPTURE_FILE} {tcpdump_filter}' - LOGGER.debug("tcpdump command: " + command) + LOGGER.debug('tcpdump command: ' + command) process = subprocess.Popen(command, universal_newlines=True, @@ -68,9 +68,9 @@ def _exec_tcpdump(self, tcpdump_filter): stderr=subprocess.PIPE) text = str(process.stdout.read()).rstrip() - LOGGER.debug("tcpdump response: " + text) + LOGGER.debug('tcpdump response: ' + text) if text: - return text.split("\n") + return text.split('\n') return [] diff --git a/test_orc/modules/dns/python/src/run.py b/test_orc/modules/dns/python/src/run.py index 06b8aa571..4cd991804 100644 --- a/test_orc/modules/dns/python/src/run.py +++ b/test_orc/modules/dns/python/src/run.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""Run DNS test module""" import argparse import signal import sys @@ -13,7 +12,7 @@ class DNSModuleRunner: - + """Run the DNS module tests.""" def __init__(self, module): signal.signal(signal.SIGINT, self._handler) @@ -33,7 +32,7 @@ def add_logger(self, module): global LOGGER LOGGER = logger.get_logger(LOG_NAME, module) - def _handler(self, signum, *other): + def _handler(self, signum): LOGGER.debug("SigtermEnum: " + str(signal.SIGTERM)) LOGGER.debug("Exit signal received: " + str(signum)) if signum in (2, signal.SIGTERM): @@ -42,7 +41,7 @@ def _handler(self, signum, *other): sys.exit(1) -def run(argv): +def run(): parser = argparse.ArgumentParser( description="Test Module DNS", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -60,4 +59,4 @@ def run(argv): if __name__ == "__main__": - run(sys.argv) + run() diff --git a/test_orc/modules/nmap/python/src/run.py b/test_orc/modules/nmap/python/src/run.py index 78eb601bb..959e30f87 100644 --- a/test_orc/modules/nmap/python/src/run.py +++ b/test_orc/modules/nmap/python/src/run.py @@ -1,4 +1,4 @@ -"""Run NMAP module""" +"""Run NMAP test module""" import argparse import signal import sys From 7fccfbb3e3191824ec05f7b7db104ccfc8fd69ce Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Fri, 26 May 2023 12:02:06 -0600 Subject: [PATCH 12/12] ovs module formatting --- .../network/modules/ovs/python/src/logger.py | 9 ++- .../modules/ovs/python/src/ovs_control.py | 58 +++++++++---------- net_orc/network/modules/ovs/python/src/run.py | 19 +++--- .../network/modules/ovs/python/src/util.py | 22 +++---- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/net_orc/network/modules/ovs/python/src/logger.py b/net_orc/network/modules/ovs/python/src/logger.py index 566a5c75e..23e697e43 100644 --- a/net_orc/network/modules/ovs/python/src/logger.py +++ b/net_orc/network/modules/ovs/python/src/logger.py @@ -1,14 +1,13 @@ -#!/usr/bin/env python3 - +"""Sets up the logger to be used for the ovs modules.""" import logging LOGGERS = {} -_LOG_FORMAT = "%(asctime)s %(name)-8s %(levelname)-7s %(message)s" +_LOG_FORMAT = '%(asctime)s %(name)-8s %(levelname)-7s %(message)s' _DATE_FORMAT = '%b %02d %H:%M:%S' # Set level to debug if set as runtime flag -logging.basicConfig(format=_LOG_FORMAT, - datefmt=_DATE_FORMAT, +logging.basicConfig(format=_LOG_FORMAT, + datefmt=_DATE_FORMAT, level=logging.INFO) def get_logger(name): diff --git a/net_orc/network/modules/ovs/python/src/ovs_control.py b/net_orc/network/modules/ovs/python/src/ovs_control.py index 53406cef2..765c50f92 100644 --- a/net_orc/network/modules/ovs/python/src/ovs_control.py +++ b/net_orc/network/modules/ovs/python/src/ovs_control.py @@ -1,32 +1,31 @@ -#!/usr/bin/env python3 - +"""OVS Control Module""" import json import logger import util -CONFIG_FILE = "/ovs/conf/system.json" -DEVICE_BRIDGE = "tr-d" -INTERNET_BRIDGE = "tr-c" +CONFIG_FILE = '/ovs/conf/system.json' +DEVICE_BRIDGE = 'tr-d' +INTERNET_BRIDGE = 'tr-c' LOGGER = logger.get_logger('ovs_ctrl') class OVSControl: - + """OVS Control""" def __init__(self): self._int_intf = None self._dev_intf = None self._load_config() def add_bridge(self, bridge_name): - LOGGER.info("Adding OVS Bridge: " + bridge_name) + LOGGER.info('Adding OVS Bridge: ' + bridge_name) # Create the bridge using ovs-vsctl commands # Uses the --may-exist option to prevent failures # if this bridge already exists by this name it won't fail # and will not modify the existing bridge - success=util.run_command("ovs-vsctl --may-exist add-br " + bridge_name) + success=util.run_command('ovs-vsctl --may-exist add-br ' + bridge_name) return success def add_port(self,port, bridge_name): - LOGGER.info("Adding Port " + port + " to OVS Bridge: " + bridge_name) + LOGGER.info('Adding Port ' + port + ' to OVS Bridge: ' + bridge_name) # Add a port to the bridge using ovs-vsctl commands # Uses the --may-exist option to prevent failures # if this port already exists on the bridge and will not @@ -36,7 +35,7 @@ def add_port(self,port, bridge_name): return success def create_net(self): - LOGGER.info("Creating baseline network") + LOGGER.info('Creating baseline network') # Create data plane self.add_bridge(DEVICE_BRIDGE) @@ -45,7 +44,7 @@ def create_net(self): self.add_bridge(INTERNET_BRIDGE) # Remove IP from internet adapter - self.set_interface_ip(self._int_intf,"0.0.0.0") + self.set_interface_ip(self._int_intf,'0.0.0.0') # Add external interfaces to data and control plane self.add_port(self._dev_intf,DEVICE_BRIDGE) @@ -56,48 +55,49 @@ def create_net(self): self.set_bridge_up(INTERNET_BRIDGE) def delete_bridge(self,bridge_name): - LOGGER.info("Deleting OVS Bridge: " + bridge_name) + LOGGER.info('Deleting OVS Bridge: ' + bridge_name) # Delete the bridge using ovs-vsctl commands # Uses the --if-exists option to prevent failures # if this bridge does not exists - success=util.run_command("ovs-vsctl --if-exists del-br " + bridge_name) + success=util.run_command('ovs-vsctl --if-exists del-br ' + bridge_name) return success def _load_config(self): - LOGGER.info("Loading Configuration: " + CONFIG_FILE) - config_json = json.load(open(CONFIG_FILE, "r", encoding="utf-8")) - self._int_intf = config_json["internet_intf"] - self._dev_intf = config_json["device_intf"] - LOGGER.info("Configuration Loaded") - LOGGER.info("Internet Interface: " + self._int_intf) - LOGGER.info("Device Interface: " + self._dev_intf) + LOGGER.info('Loading Configuration: ' + CONFIG_FILE) + with open(CONFIG_FILE, 'r', encoding='utf-8') as conf_file: + config_json = json.load(conf_file) + self._int_intf = config_json['internet_intf'] + self._dev_intf = config_json['device_intf'] + LOGGER.info('Configuration Loaded') + LOGGER.info('Internet Interface: ' + self._int_intf) + LOGGER.info('Device Interface: ' + self._dev_intf) def restore_net(self): - LOGGER.info("Restoring Network...") + LOGGER.info('Restoring Network...') # Delete data plane self.delete_bridge(DEVICE_BRIDGE) # Delete control plane self.delete_bridge(INTERNET_BRIDGE) - LOGGER.info("Network is restored") + LOGGER.info('Network is restored') def show_config(self): - LOGGER.info("Show current config of OVS") - success=util.run_command("ovs-vsctl show") + LOGGER.info('Show current config of OVS') + success=util.run_command('ovs-vsctl show') return success def set_bridge_up(self,bridge_name): - LOGGER.info("Setting Bridge device to up state: " + bridge_name) - success=util.run_command("ip link set dev " + bridge_name + " up") + LOGGER.info('Setting Bridge device to up state: ' + bridge_name) + success=util.run_command('ip link set dev ' + bridge_name + ' up') return success def set_interface_ip(self,interface, ip_addr): - LOGGER.info("Setting interface " + interface + " to " + ip_addr) + LOGGER.info('Setting interface ' + interface + ' to ' + ip_addr) # Remove IP from internet adapter - util.run_command("ifconfig " + interface + " 0.0.0.0") + util.run_command('ifconfig ' + interface + ' 0.0.0.0') -if __name__ == "__main__": +if __name__ == '__main__': ovs = OVSControl() ovs.create_net() ovs.show_config() diff --git a/net_orc/network/modules/ovs/python/src/run.py b/net_orc/network/modules/ovs/python/src/run.py index f91c2dfeb..5787a74e6 100644 --- a/net_orc/network/modules/ovs/python/src/run.py +++ b/net_orc/network/modules/ovs/python/src/run.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 - +"""Run OVS module""" import logger import signal import sys @@ -10,7 +9,7 @@ LOGGER = logger.get_logger('ovs_control_run') class OVSControlRun: - + """Run the OVS module.""" def __init__(self): signal.signal(signal.SIGINT, self.handler) @@ -18,7 +17,7 @@ def __init__(self): signal.signal(signal.SIGABRT, self.handler) signal.signal(signal.SIGQUIT, self.handler) - LOGGER.info("Starting OVS Control") + LOGGER.info('Starting OVS Control') # Get all components ready self._ovs_control = OVSControl() @@ -30,11 +29,11 @@ def __init__(self): self._ovs_control.show_config() # Get network ready (via Network orchestrator) - LOGGER.info("Network is ready. Waiting for device information...") + LOGGER.info('Network is ready. Waiting for device information...') #Loop forever until process is stopped while True: - LOGGER.info("OVS Running") + LOGGER.info('OVS Running') time.sleep(1000) # TODO: This time should be configurable (How long to hold before exiting, @@ -44,11 +43,11 @@ def __init__(self): # Tear down network #self._ovs_control.shutdown() - def handler(self, signum, frame): - LOGGER.info("SigtermEnum: " + str(signal.SIGTERM)) - LOGGER.info("Exit signal received: " + str(signum)) + def handler(self, signum): + LOGGER.info('SigtermEnum: ' + str(signal.SIGTERM)) + LOGGER.info('Exit signal received: ' + str(signum)) if (signum == 2 or signal == signal.SIGTERM): - LOGGER.info("Exit signal received. Restoring network...") + LOGGER.info('Exit signal received. Restoring network...') self._ovs_control.shutdown() sys.exit(1) diff --git a/net_orc/network/modules/ovs/python/src/util.py b/net_orc/network/modules/ovs/python/src/util.py index c9eba39ff..a3ebbb10a 100644 --- a/net_orc/network/modules/ovs/python/src/util.py +++ b/net_orc/network/modules/ovs/python/src/util.py @@ -1,21 +1,23 @@ +"""Provides basic utilities for a ovs module.""" import subprocess import logger +LOGGER = logger.get_logger('util') def run_command(cmd): success = False - LOGGER = logger.get_logger('util') - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() - if process.returncode !=0: - err_msg = "%s. Code: %s" % (stderr.strip(), process.returncode) - LOGGER.error("Command Failed: " + cmd) - LOGGER.error("Error: " + err_msg) + if process.returncode != 0: + err_msg = f'{stderr.strip()}. Code: {process.returncode}' + LOGGER.error('Command Failed: ' + cmd) + LOGGER.error('Error: ' + err_msg) else: - succ_msg = "%s. Code: %s" % (stdout.strip().decode('utf-8'), process.returncode) - LOGGER.info("Command Success: " + cmd) - LOGGER.info("Success: " + succ_msg) + msg = stdout.strip().decode('utf-8') + succ_msg = f'{msg}. Code: {process.returncode}' + LOGGER.info('Command Success: ' + cmd) + LOGGER.info('Success: ' + succ_msg) success = True return success