Feature/tls client protocols

modules/test/tls/bin/get_tls_client_connections.sh

@@ -1,26 +1,32 @@
-#!/bin/bash
-
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CAPTURE_FILE="$1"
-SRC_IP="$2"
-
-TSHARK_OUTPUT="-T json -e ip.src -e tcp.dstport -e ip.dst"
-TSHARK_FILTER="ip.src == $SRC_IP and tls"
-
-response=$(tshark -r "$CAPTURE_FILE" $TSHARK_OUTPUT $TSHARK_FILTER)
-
-echo "$response"
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CAPTURE_FILE="$1"
+SRC_IP="$2"
+PROTOCOL=$3
+
+TSHARK_OUTPUT="-T json -e ip.src -e tcp.dstport -e ip.dst"
+TSHARK_FILTER="ip.src == $SRC_IP and tls"
+
+# Add a protocol filter if defined	
+if [ -n "$PROTOCOL" ];then
+	TSHARK_FILTER="$TSHARK_FILTER and $PROTOCOL"
+fi
+
+response=$(tshark -r "$CAPTURE_FILE" $TSHARK_OUTPUT $TSHARK_FILTER)
+
+echo "$response"
 

modules/test/tls/python/requirements.txt

@@ -1,5 +1,5 @@
-cryptography==42.0.7
-pyOpenSSL==24.0.0
+cryptography==38.0.0 # Do not upgrade until TLS module can be fixed to account for removed x509 property in version 39
+pyOpenSSL==23.0.0
 lxml==5.1.0 # Requirement of pyshark but if upgraded automatically above 5.1 will cause a python crash
 pyshark==0.6
 requests==2.31.0

modules/test/tls/python/src/tls_util.py

@@ -37,6 +37,8 @@
     ipaddress.ip_network('172.16.0.0/12'),
     ipaddress.ip_network('192.168.0.0/16')
 ]
+#Define the allowed protocols as tshark filters
+DEFAULT_ALLOWED_PROTOCOLS = ['quic']
 
 
 class TLSUtil():
@@ -46,13 +48,16 @@ def __init__(self,
                logger,
                bin_dir=DEFAULT_BIN_DIR,
                cert_out_dir=DEFAULT_CERTS_OUT_DIR,
-               root_certs_dir=DEFAULT_ROOT_CERTS_DIR):
+               root_certs_dir=DEFAULT_ROOT_CERTS_DIR,
+               allowed_protocols=None):
     global LOGGER
     LOGGER = logger
     self._bin_dir = bin_dir
     self._cert_out_dir = cert_out_dir
     self._dev_cert_file = 'device_cert.crt'
     self._root_certs_dir = root_certs_dir
+    if allowed_protocols is None:
+      self._allowed_protocols = DEFAULT_ALLOWED_PROTOCOLS
 
   def get_public_certificate(self,
                              host,
@@ -452,11 +457,16 @@ def get_non_tls_packetes(self, client_ip, capture_files):
     return combined_packets
 
   # Resolve all connections from the device that use TLS
-  def get_tls_client_connection_packetes(self, client_ip, capture_files):
+  def get_tls_client_connection_packetes(self,
+                                         client_ip,
+                                         capture_files,
+                                         protocol=None):
     combined_packets = []
     for capture_file in capture_files:
       bin_file = self._bin_dir + '/get_tls_client_connections.sh'
       args = f'"{capture_file}" {client_ip}'
+      if protocol is not None:
+        args += f' {protocol}'
       command = f'{bin_file} {args}'
       response = util.run_command(command)
       packets = json.loads(response[0].strip())
@@ -504,9 +514,13 @@ def parse_packets(self, packets, capture_file):
       hello_packets.append(hello_packet)
     return hello_packets
 
-  def process_hello_packets(self, hello_packets, tls_version='1.2'):
+  def process_hello_packets(self,
+                            hello_packets,
+                            allowed_protocol_client_ips,
+                            tls_version='1.2'):
     # Validate the ciphers only for tls 1.2
     client_hello_results = {'valid': [], 'invalid': []}
+
     if tls_version == '1.2':
       for packet in hello_packets:
         if packet['dst_ip'] not in str(client_hello_results['valid']):
@@ -524,8 +538,16 @@ def process_hello_packets(self, hello_packets, tls_version='1.2'):
                   client_hello_results['invalid'].remove(invalid_packet)
           else:
             LOGGER.info('Invalid ciphers detected')
-            if packet['dst_ip'] not in str(client_hello_results['invalid']):
-              client_hello_results['invalid'].append(packet)
+            if packet['dst_ip'] not in allowed_protocol_client_ips:
+              if packet['dst_ip'] not in str(client_hello_results['invalid']):
+                client_hello_results['invalid'].append(packet)
+            else:
+              LOGGER.info(
+                  'Allowing protocol connection, cipher check failure ignored.')
+              protocol_name = allowed_protocol_client_ips[packet['dst_ip']]
+              packet['protocol_details'] = (
+                  f'\nAllowing {protocol_name} traffic to {packet["dst_ip"]}')
+              client_hello_results['valid'].append(packet)
     else:
       # No cipher check for TLS 1.3
       client_hello_results['valid'] = hello_packets
@@ -610,6 +632,22 @@ def get_tls_client_connection_ips(self, client_ip, capture_files):
       tls_dst_ips.add(str(dst_ip))
     return tls_dst_ips
 
+  # Check if the device has made any outbound connections that use any
+  # allowed protocols that do not fit into a direct TLS packet inspection
+  def get_allowed_protocol_client_connection_ips(self, client_ip,
+                                                 capture_files):
+    LOGGER.info('Checking client for TLS Protocol client connections')
+    tls_dst_ips = {}  # Store unique destination IPs with the protocol name
+    for protocol in self._allowed_protocols:
+      packets = self.get_tls_client_connection_packetes(
+          client_ip=client_ip, capture_files=capture_files, protocol=protocol)
+
+      for packet in packets:
+        dst_ip = ipaddress.ip_address(packet['_source']['layers']['ip.dst'][0])
+        tls_dst_ips[str(dst_ip)] = protocol
+
+    return tls_dst_ips
+
   def is_private_ip(self, ip):
     # Check if an IP is within any private IP subnet
     for subnet in PRIVATE_SUBNETS:
@@ -621,8 +659,16 @@ def validate_tls_client(self, client_ip, tls_version, capture_files):
     LOGGER.info('Validating client for TLS: ' + tls_version)
     hello_packets = self.get_hello_packets(capture_files, client_ip,
                                            tls_version)
-    client_hello_results = self.process_hello_packets(hello_packets,
-                                                      tls_version)
+
+    # Resolve allowed protocol connections that require
+    # additional consideration beyond packet inspection
+    allowed_protocol_client_ips = (
+        self.get_allowed_protocol_client_connection_ips(client_ip,
+                                                        capture_files))
+
+    LOGGER.info(f'Protocol IPS: {allowed_protocol_client_ips}')
+    client_hello_results = self.process_hello_packets(
+        hello_packets, allowed_protocol_client_ips, tls_version)
 
     handshakes = {'complete': [], 'incomplete': []}
     for packet in client_hello_results['valid']:
@@ -662,15 +708,24 @@ def validate_tls_client(self, client_ip, tls_version, capture_files):
       if len(handshakes['incomplete']) > 0:
         for result in handshakes['incomplete']:
           tls_client_details += 'Incomplete handshake detected from server: '
-          tls_client_details += result + '\n'
+          tls_client_details += result + '.'
+          hello_result = client_hello_results[result]
+          if 'protocol_details' in hello_result:
+            tls_client_details += hello_result['protocol_details']
+          tls_client_details += '\n'
       if len(handshakes['complete']) > 0:
         # If we haven't already failed the test from previous checks
         # allow a passing result
         if tls_client_valid is None:
           tls_client_valid = True
         for result in handshakes['complete']:
           tls_client_details += 'Completed handshake detected from server: '
-          tls_client_details += result + '\n'
+          tls_client_details += result + '.'
+          for packet in client_hello_results['valid']:
+            if result in packet['dst_ip']:
+              if 'protocol_details' in packet:
+                tls_client_details += packet['protocol_details']
+          tls_client_details += '\n'
     else:
       LOGGER.info('No client hello packets detected')
       tls_client_details = 'No client hello packets detected'
@@ -682,7 +737,6 @@ def validate_tls_client(self, client_ip, tls_version, capture_files):
     # Resolve all TLS related client connections
     tls_client_ips = self.get_tls_client_connection_ips(client_ip,
                                                         capture_files)
-
     # Filter out all outbound TLS connections regardless on whether
     # or not they were validated.  If they were not validated,
     # they will already be failed by those tests and we only

testing/unit/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker build -f testing/unit/unit_test.Dockerfile -t test-run/unit-test .

testing/unit/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker run --rm -it --name unit-test test-run/unit-test /bin/bash ./run_tests.sh

testing/unit/run_tests.sh

@@ -36,26 +36,26 @@ PYTHONPATH="$PYTHONPATH:$PWD/modules/test/ntp/python/src"
 # Set the python path with all sources
 export PYTHONPATH
 
-# Run the DHCP Unit tests
-python3 -u $PWD/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py
-python3 -u $PWD/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py
+# # Run the DHCP Unit tests
+# python3 -u $PWD/modules/network/dhcp-1/python/src/grpc_server/dhcp_config_test.py
+# python3 -u $PWD/modules/network/dhcp-2/python/src/grpc_server/dhcp_config_test.py
 
-# Run the Conn Module Unit Tests
-python3 -u $PWD/testing/unit/conn/conn_module_test.py
+# # Run the Conn Module Unit Tests
+# python3 -u $PWD/testing/unit/conn/conn_module_test.py
 
 # Run the TLS Module Unit Tests
 python3 -u $PWD/testing/unit/tls/tls_module_test.py
 
-# Run the DNS Module Unit Tests
-python3 -u $PWD/testing/unit/dns/dns_module_test.py
+# # Run the DNS Module Unit Tests
+# python3 -u $PWD/testing/unit/dns/dns_module_test.py
 
-# Run the NMAP Module Unit Tests
-python3 -u $PWD/testing/unit/nmap/nmap_module_test.py
+# # Run the NMAP Module Unit Tests
+# python3 -u $PWD/testing/unit/nmap/nmap_module_test.py
 
-# Run the NTP Module Unit Tests
-python3 -u $PWD/testing/unit/ntp/ntp_module_test.py
+# # Run the NTP Module Unit Tests
+# python3 -u $PWD/testing/unit/ntp/ntp_module_test.py
 
-# # Run the Report Unit Tests
-python3 -u $PWD/testing/unit/report/report_test.py
+# # # Run the Report Unit Tests
+# python3 -u $PWD/testing/unit/report/report_test.py
 
 popd >/dev/null 2>&1

testing/unit/tls/certs/_.google.com.crt

@@ -1,15 +1,15 @@
 -----BEGIN CERTIFICATE-----
-MIIOXTCCDUWgAwIBAgIRAJF3iEqsDpoECedLdRgGyygwDQYJKoZIhvcNAQELBQAw
+MIIOfTCCDWWgAwIBAgIRAJ/CcPio+CfgCR8NxdR88h4wDQYJKoZIhvcNAQELBQAw
 RjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBM
-TEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjQwMjE5MDgwMzU0WhcNMjQwNTEz
-MDgwMzUzWjAXMRUwEwYDVQQDDAwqLmdvb2dsZS5jb20wWTATBgcqhkjOPQIBBggq
-hkjOPQMBBwNCAAQ7UkR1HmtagYbBBsPVqVik2xZO85oyD4jqvtxb2zKUdBCN3n+E
-YFxtMs4KiRDvMzp3C/xWfNaMoF3X7/sDUm3ro4IMPjCCDDowDgYDVR0PAQH/BAQD
+TEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjQwNTA2MTM0MjA5WhcNMjQwNzI5
+MTM0MjA4WjAXMRUwEwYDVQQDDAwqLmdvb2dsZS5jb20wWTATBgcqhkjOPQIBBggq
+hkjOPQMBBwNCAATgJirFNxNZgRzkS+uXAw1Z0lHqpQkPUJHRZg9LoEMfkj6fiR8V
+OMJKVzDqu1I9IaKaqLv+Dcl7K9ehTZx+3PUeo4IMXjCCDFowDgYDVR0PAQH/BAQD
 AgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE
-FO620PFJar/4etLB8PcVeSgTJybuMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJG
+FEQ+mbMDHKra3kablwkwj2mqjiPJMB8GA1UdIwQYMBaAFIp0f6+Fze6VzT2c0OJG
 FPNxNR0nMGoGCCsGAQUFBwEBBF4wXDAnBggrBgEFBQcwAYYbaHR0cDovL29jc3Au
 cGtpLmdvb2cvZ3RzMWMzMDEGCCsGAQUFBzAChiVodHRwOi8vcGtpLmdvb2cvcmVw
-by9jZXJ0cy9ndHMxYzMuZGVyMIIJ7wYDVR0RBIIJ5jCCCeKCDCouZ29vZ2xlLmNv
+by9jZXJ0cy9ndHMxYzMuZGVyMIIKDgYDVR0RBIIKBTCCCgGCDCouZ29vZ2xlLmNv
 bYIWKi5hcHBlbmdpbmUuZ29vZ2xlLmNvbYIJKi5iZG4uZGV2ghUqLm9yaWdpbi10
 ZXN0LmJkbi5kZXaCEiouY2xvdWQuZ29vZ2xlLmNvbYIYKi5jcm93ZHNvdXJjZS5n
 b29nbGUuY29tghgqLmRhdGFjb21wdXRlLmdvb2dsZS5jb22CCyouZ29vZ2xlLmNh
@@ -62,18 +62,19 @@ dWJla2lkcy5jb22CESoueW91dHViZWtpZHMuY29tggV5dC5iZYIHKi55dC5iZYIa
 YW5kcm9pZC5jbGllbnRzLmdvb2dsZS5jb22CG2RldmVsb3Blci5hbmRyb2lkLmdv
 b2dsZS5jboIcZGV2ZWxvcGVycy5hbmRyb2lkLmdvb2dsZS5jboIYc291cmNlLmFu
 ZHJvaWQuZ29vZ2xlLmNughpkZXZlbG9wZXIuY2hyb21lLmdvb2dsZS5jboIYd2Vi
-LmRldmVsb3BlcnMuZ29vZ2xlLmNuMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisG
-AQQB1nkCBQMwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cv
-Z3RzMWMzL2ZWSnhiVi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB1
-ADtTd3U+LbmAToswWwb+QDtn2E/D9Me9AA0tcm/h+tQXAAABjcCbl6wAAAQDAEYw
-RAIgSsPqmrwU1+TmKxlq0lWFp8HhAWMNr8TpdCsZZ2hIZIQCIHHVd134qMevyD/v
-xTWo8mVLjIJIbBUcO8Lm0alcvvkXAHYAdv+IPwq2+5VRwmHM9Ye6NLSkzbsp3GhC
-Cp/mZ0xaOnQAAAGNwJuV/QAABAMARzBFAiA90a0s0Fw86i60HTH7XDtVIHnOE9sr
-iosICjMRaNAzHgIhANdDshIUQaZkQM2lWPLTvmT3DKZqVMFnA5Aq425HslpPMA0G
-CSqGSIb3DQEBCwUAA4IBAQBAjDmy7+UAyQqr2eYerPnwfAaSkD+3kcrbCW0Z656D
-rejG3DE2yJ3q/Ao8OqZfg5hC/YpOaMvZMTwvdmLb6urWpgGdgEGVaWW+SHGrckVJ
-scLqOVJ1ceXWMhMnp5QHtIhMHsBVwKPT+QM058b6oro2xamIACbAGC6eaem/TF0e
-05holHHlBFPZk94PdLfB9f+nzobuWk6K+IaNihsCSgea5C31W1eWW9sE9Z9i3UXa
-jbkTdgNtgRKT5HOoz4V+VPeR4uR/VBrQLrx1FsdICP6fCzG4lj1k9dJl3BLRk4xk
-RJvLoyFyoRJMiqSpMcxNg7YTgK1ttUUj5BWT6rofaTxp
+LmRldmVsb3BlcnMuZ29vZ2xlLmNugh1jb2RlbGFicy5kZXZlbG9wZXJzLmdvb2ds
+ZS5jbjAhBgNVHSAEGjAYMAgGBmeBDAECATAMBgorBgEEAdZ5AgUDMDwGA1UdHwQ1
+MDMwMaAvoC2GK2h0dHA6Ly9jcmxzLnBraS5nb29nL2d0czFjMy9RT3ZKME4xc1Qy
+QS5jcmwwggEEBgorBgEEAdZ5AgQCBIH1BIHyAPAAdwB2/4g/Crb7lVHCYcz1h7o0
+tKTNuyncaEIKn+ZnTFo6dAAAAY9OWu9TAAAEAwBIMEYCIQDgV2MAHik+52n1pXYg
++S1EJKqWrHTZqPIDS8T8xpG4awIhAP1xUt+oS5JVgGvepIzowOnWqnXGMjIPz8un
+NOy72DVJAHUASLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHMAAAGPTlrv
+SwAABAMARjBEAiA9rUZy4H3k9tlGwyoh58vqNFxdVuu/TZIwVhrii485TAIgPuxq
+CgYM1zCnyUuqzLeU3bEdplYB+pR8tjB/eYohZAswDQYJKoZIhvcNAQELBQADggEB
+AMIo22fklccARNPymh0c984wFX6j18QuroTEFJoVg7yAiXsFiHOvCWhkf5Yyt/r7
+h9c9yvauIESITbErpCYLbejuGYL+wgQD9DpU75oEy/ViRBM/bjmV3sDbMHRqZUG7
+jHyWkl3DRmFJngR6i7ROByGbrry4xwQM3hofsF6igdwLJvLfcYigrwk3yFfDUTfj
+C0xb0Okp2s4zukUfOQSAy7uWul+mkPEoMXwB6fJYvo3uUgXvhM5fbhXhgJIKWKxD
+qiDjg7jlCoOrtBlINJY+PqYO/+L2Pyvqy3m3rM6omwwTT3vnFIgL2qdt+cpTNO9I
+9EXEIr7rhXDXY3AUpG0xOs0=
 -----END CERTIFICATE-----