diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index fbdbe442c..c981dbd56 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -7,7 +7,7 @@ on:
     - cron: '0 13 * * *'
 
 jobs:
-  testrun:
+  testrun_baseline:
     name:  Baseline
     runs-on: ubuntu-20.04
     timeout-minutes: 20
@@ -17,11 +17,21 @@ jobs:
       - name: Run tests
         shell: bash {0}
         run: testing/test_baseline
-
+    
+  testrun_tests:
+    name:  Tests
+    runs-on: ubuntu-20.04
+    timeout-minutes: 40
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.4
+      - name: Run tests
+        shell: bash {0}
+        run: testing/test_tests
   pylint:
     name:  Pylint
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
     steps:
       - name: Checkout source
         uses: actions/checkout@v2.3.4
diff --git a/testing/docker/ci_test_device1/Dockerfile b/testing/docker/ci_test_device1/Dockerfile
new file mode 100644
index 000000000..0bb697509
--- /dev/null
+++ b/testing/docker/ci_test_device1/Dockerfile
@@ -0,0 +1,11 @@
+
+FROM ubuntu:jammy
+
+#Update and get all additional requirements not contained in the base image 
+RUN apt-get update && apt-get -y upgrade
+
+RUN apt-get update && apt-get install -y isc-dhcp-client ntpdate coreutils moreutils inetutils-ping curl jq dnsutils openssl netcat-openbsd 
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
\ No newline at end of file
diff --git a/testing/docker/ci_test_device1/entrypoint.sh b/testing/docker/ci_test_device1/entrypoint.sh
new file mode 100755
index 000000000..8113704be
--- /dev/null
+++ b/testing/docker/ci_test_device1/entrypoint.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+ip a
+
+declare -A options
+for option in $*; do
+    if [[ $option == *"="* ]]; then
+        k=$(echo $option | cut -d'=' -f1)
+        v=$(echo $option | cut -d'=' -f2)
+        options[$k]=$v
+    else
+        options[$option]=$option
+    fi
+done
+
+OUT=/out/testrun_ci.json
+
+NTP_SERVER=10.10.10.5
+DNS_SERVER=10.10.10.4
+
+function wout(){
+    temp=${1//./\".\"}
+    key=${temp:1}\"
+    echo $key
+    value=$2
+    jq "$key+=\"$value\"" $OUT | sponge $OUT
+}
+
+
+dig @8.8.8.8 +short www.google.com
+
+# DHCP
+ip addr flush dev eth0
+PID_FILE=/var/run/dhclient.pid
+if [ -f $PID_FILE ]; then
+    kill -9 $(cat $PID_FILE) || true
+    rm -f $PID_FILE
+fi
+dhclient -v eth0
+
+
+if [ -n "${options[oddservices]}" ]; then
+    echo Running services on non standard ports and open default ports 
+
+    echo Starting FTP 21514 and open default 20,21
+    nc -nvlt -p 20 &
+    nc -nvlt -p 21 &
+    (while true; do echo -e "220 ProFTPD 1.3.5e Server (Debian) $(hostname)" | nc -l -w 1 21514; done) &
+    
+    echo Starting SMTP 1256 and open default 25, 465, 587
+    nc -nvlt -p 25 &
+    nc -nvlt -p 465 &
+    nc -nvlt -p 587 &
+    (while true; do echo -e "220 $(hostname) ESMTP Postfix (Ubuntu)" | nc -l -w 1 1256; done) &
+
+    echo Starting IMAP 5361 and open default ports 143, 993
+    nc -nvlt -p 143 &
+    nc -nvlt -p 993 &
+    (while true; do echo -e "* OK [CAPABILITY IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE STARTTLS AUTH=PLAIN] Dovecot (Ubuntu) ready.\r\n" \
+        | nc -l -w 1 5361; done) &
+
+    echo Starting POP3 23451 and open default 110, 995
+    nc -nvlt -p 110 &
+    nc -nvlt -p 995 &
+    (while true; do echo -ne "+OK POP3 Server ready\r\n" | nc -l -w 1 23451; done) &
+
+    echo starting TFTP UDP 69
+    (while true; do echo -ne "\0\x05\0\0\x07\0" | nc -u -l -w 1 69; done) &
+
+fi
+
+if [ -n "${options[snmp]}" ]; then
+    echo starting mock none snmpv3 on port UDP 161
+    (while true; do echo -ne " \x02\x01\ " | nc -u -l -w 1 161; done) &
+fi
+
+if [ -n "${options[snmpv3]}" ]; then
+    echo starting mock SNMPv3 UDP 161
+    (while true; do echo -ne "  \x02\x01\x030 \x02\x02Ji\x02    \x04\x01 \x02\x01\x03\x04" | nc -u -l -w 1 161; done) &
+fi
+
+if [ -n "${options[ssh]}" ]; then
+    echo Starting SSH server
+    /usr/local/sbin/sshd
+elif [ -n "${options[sshv1]}" ]; then
+    echo Starting SSHv1 server
+    echo 'Protocol 1' >> /usr/local/etc/sshd_config
+    /usr/local/sbin/sshd
+fi
+
+tail -f /dev/null
\ No newline at end of file
diff --git a/testing/example/mac b/testing/example/mac
new file mode 100644
index 000000000..e69de29bb
diff --git a/testing/example/mac1/results.json b/testing/example/mac1/results.json
new file mode 100644
index 000000000..e1b837225
--- /dev/null
+++ b/testing/example/mac1/results.json
@@ -0,0 +1,252 @@
+{
+    "device": {
+      "mac_addr": "7e:41:12:d2:35:6a"
+    },
+    "dns": {
+      "results": [
+        {
+          "name": "dns.network.from_device",
+          "description": "Verify the device sends DNS requests",
+          "expected_behavior": "The device sends DNS requests.",
+          "start": "2023-07-03T13:35:48.990574",
+          "result": "compliant",
+          "end": "2023-07-03T13:35:49.035528",
+          "duration": "0:00:00.044954"
+        },
+        {
+          "name": "dns.network.from_dhcp",
+          "description": "Verify the device allows for a DNS server to be entered automatically",
+          "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server",
+          "start": "2023-07-03T13:35:49.035701",
+          "result": "non-compliant",
+          "end": "2023-07-03T13:35:49.041532",
+          "duration": "0:00:00.005831"
+        },
+        {
+          "name": "dns.mdns",
+          "description": "If the device has MDNS (or any kind of IP multicast), can it be disabled",
+          "start": "2023-07-03T13:35:49.041679",
+          "result": "non-compliant",
+          "end": "2023-07-03T13:35:49.057430",
+          "duration": "0:00:00.015751"
+        }
+      ]
+    },
+    "nmap": {
+      "results": [
+        {
+          "name": "security.nmap.ports",
+          "description": "Run an nmap scan of open ports",
+          "expected_behavior": "Report all open ports",
+          "config": {
+            "security.services.ftp": {
+              "tcp_ports": {
+                "20": {
+                  "allowed": false,
+                  "description": "File Transfer Protocol (FTP) Server Data Transfer",
+                  "result": "compliant"
+                },
+                "21": {
+                  "allowed": false,
+                  "description": "File Transfer Protocol (FTP) Server Data Transfer",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check FTP port 20/21 is disabled and FTP is not running on any port",
+              "expected_behavior": "There is no FTP service running on any port"
+            },
+            "security.services.ssh": {
+              "tcp_ports": {
+                "22": {
+                  "allowed": true,
+                  "description": "Secure Shell (SSH) server",
+                  "version": "2.0",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check TELNET port 23 is disabled and TELNET is not running on any port",
+              "expected_behavior": "There is no FTP service running on any port"
+            },
+            "security.services.telnet": {
+              "tcp_ports": {
+                "23": {
+                  "allowed": false,
+                  "description": "Telnet Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check TELNET port 23 is disabled and TELNET is not running on any port",
+              "expected_behavior": "There is no FTP service running on any port"
+            },
+            "security.services.smtp": {
+              "tcp_ports": {
+                "25": {
+                  "allowed": false,
+                  "description": "Simple Mail Transfer Protocol (SMTP) Server",
+                  "result": "compliant"
+                },
+                "465": {
+                  "allowed": false,
+                  "description": "Simple Mail Transfer Protocol over SSL (SMTPS) Server",
+                  "result": "compliant"
+                },
+                "587": {
+                  "allowed": false,
+                  "description": "Simple Mail Transfer Protocol via TLS (SMTPS) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check SMTP port 25 is disabled and ports 465 or 587 with SSL encryption are (not?) enabled and SMTP is not running on any port.",
+              "expected_behavior": "There is no smtp service running on any port"
+            },
+            "security.services.http": {
+              "tcp_ports": {
+                "80": {
+                  "service_scan": {
+                    "script": "http-methods"
+                  },
+                  "allowed": false,
+                  "description": "Administrative Insecure Web-Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check that there is no HTTP server running on any port",
+              "expected_behavior": "Device is unreachable on port 80 (or any other port) and only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)"
+            },
+            "security.services.pop": {
+              "tcp_ports": {
+                "110": {
+                  "allowed": false,
+                  "description": "Post Office Protocol v3 (POP3) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check POP port 110 is disalbed and POP is not running on any port",
+              "expected_behavior": "There is no pop service running on any port"
+            },
+            "security.services.imap": {
+              "tcp_ports": {
+                "143": {
+                  "allowed": false,
+                  "description": "Internet Message Access Protocol (IMAP) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check IMAP port 143 is disabled and IMAP is not running on any port",
+              "expected_behavior": "There is no imap service running on any port"
+            },
+            "security.services.snmpv3": {
+              "tcp_ports": {
+                "161": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP)",
+                  "result": "compliant"
+                },
+                "162": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP) Trap",
+                  "result": "compliant"
+                }
+              },
+              "udp_ports": {
+                "161": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP)"
+                },
+                "162": {
+                  "allowed": false,
+                  "description": "Simple Network Management Protocol (SNMP) Trap"
+                }
+              },
+              "description": "Check SNMP port 161/162 is disabled.  If SNMP is an essential service, check it supports version 3",
+              "expected_behavior": "Device is unreachable on port 161 (or any other port) and device is unreachable on port 162 (or any other port) unless SNMP is essential in which case it is SNMPv3 is used."
+            },
+            "security.services.https": {
+              "tcp_ports": {
+                "80": {
+                  "allowed": false,
+                  "description": "Administrative Secure Web-Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check that if there is a web server running it is running on a secure port.",
+              "expected_behavior": "Device only responds to HTTPS requests on port 443 (or any other port if HTTP is used at all)"
+            },
+            "security.services.vnc": {
+              "tcp_ports": {
+                "5800": {
+                  "allowed": false,
+                  "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol Over HTTP",
+                  "result": "compliant"
+                },
+                "5500": {
+                  "allowed": false,
+                  "description": "Virtual Network Computing (VNC) Remote Frame Buffer Protocol",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check VNC is disabled on any port",
+              "expected_behavior": "Device cannot be accessed /connected to via VNc on any port"
+            },
+            "security.services.tftp": {
+              "udp_ports": {
+                "69": {
+                  "allowed": false,
+                  "description": "Trivial File Transfer Protocol (TFTP) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check TFTP port 69 is disabled (UDP)",
+              "expected_behavior": "There is no tftp service running on any port"
+            },
+            "security.services.ntp": {
+              "udp_ports": {
+                "123": {
+                  "allowed": false,
+                  "description": "Network Time Protocol (NTP) Server",
+                  "result": "compliant"
+                }
+              },
+              "description": "Check NTP port 123 is disabled and the device is not operating as an NTP server",
+              "expected_behavior": "The device dos not respond to NTP requests when it's IP is set as the NTP server on another device"
+            }
+          },
+          "start": "2023-07-03T13:36:26.923704",
+          "result": "compliant",
+          "end": "2023-07-03T13:36:52.965535",
+          "duration": "0:00:26.041831"
+        }
+      ]
+    },
+    "baseline": {
+      "results": [
+        {
+          "name": "baseline.pass",
+          "description": "Simulate a compliant test",
+          "expected_behavior": "A compliant test result is generated",
+          "start": "2023-07-03T13:37:29.100681",
+          "result": "compliant",
+          "end": "2023-07-03T13:37:29.100869",
+          "duration": "0:00:00.000188"
+        },
+        {
+          "name": "baseline.fail",
+          "description": "Simulate a non-compliant test",
+          "expected_behavior": "A non-compliant test result is generated",
+          "start": "2023-07-03T13:37:29.100961",
+          "result": "non-compliant",
+          "end": "2023-07-03T13:37:29.101089",
+          "duration": "0:00:00.000128"
+        },
+        {
+          "name": "baseline.skip",
+          "description": "Simulate a skipped test",
+          "expected_behavior": "A skipped test result is generated",
+          "start": "2023-07-03T13:37:29.101164",
+          "result": "skipped",
+          "end": "2023-07-03T13:37:29.101283",
+          "duration": "0:00:00.000119"
+        }
+      ]
+    }
+  }
\ No newline at end of file
diff --git a/testing/test_baseline b/testing/test_baseline
index f12d124de..2b95ded23 100755
--- a/testing/test_baseline
+++ b/testing/test_baseline
@@ -20,7 +20,7 @@ ifconfig
 
 # Setup requirements
 sudo apt-get update
-sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils
+sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils isc-dhcp-client
 
 pip3 install pytest
 
@@ -80,6 +80,6 @@ echo "Done baseline test"
 
 more $TESTRUN_OUT
 
-pytest testing/
+pytest testing/test_baseline.py
 
 exit $?
\ No newline at end of file
diff --git a/testing/test_baseline.py b/testing/test_baseline.py
index 246857581..520f909f7 100644
--- a/testing/test_baseline.py
+++ b/testing/test_baseline.py
@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+""" Test assertions for CI network baseline test """
+# Temporarily disabled because using Pytest fixtures
+# TODO refactor fixtures to not trigger error
+# pylint: disable=redefined-outer-name
+
 import json
 import pytest
 import re
@@ -24,14 +29,13 @@
 
 @pytest.fixture
 def container_data():
-  dir = os.path.dirname(os.path.abspath(__file__))
   with open(CI_BASELINE_OUT, encoding='utf-8') as f:
     return json.load(f)
 
 @pytest.fixture
 def validator_results():
-  dir = os.path.dirname(os.path.abspath(__file__))
-  with open(os.path.join(dir,
+  basedir = os.path.dirname(os.path.abspath(__file__))
+  with open(os.path.join(basedir,
                          '../',
                          'runtime/validation/faux-dev/result.json'),
                          encoding='utf-8') as f:
@@ -63,6 +67,5 @@ def test_dns_server_resolves(container_data):
 
 @pytest.mark.skip(reason='requires internet')
 def test_validator_results_compliant(validator_results):
-  results = [True if x['result'] == 'compliant' else False
-    for x in validator_results['results']]
+  results = [x['result'] == 'compliant' for x in validator_results['results']]
   assert all(results)
diff --git a/testing/test_pylint b/testing/test_pylint
index 3f4063812..3f4d8a3ed 100755
--- a/testing/test_pylint
+++ b/testing/test_pylint
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ERROR_LIMIT=100
+ERROR_LIMIT=175
 
 sudo cmd/install
 
diff --git a/testing/test_tests b/testing/test_tests
new file mode 100755
index 000000000..6ba9fef94
--- /dev/null
+++ b/testing/test_tests
@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o xtrace
+ip a
+TEST_DIR=/tmp/results
+MATRIX=testing/test_tests.json
+
+mkdir -p $TEST_DIR
+
+# Setup requirements
+sudo apt-get update
+sudo apt-get install openvswitch-common openvswitch-switch tcpdump jq moreutils coreutils isc-dhcp-client
+
+pip3 install pytest
+
+# Start OVS
+# Setup device network
+sudo ip link add dev endev0a type veth peer name endev0b
+sudo ip link set dev endev0a up
+sudo ip link set dev endev0b up
+sudo docker network create -d macvlan -o parent=endev0b endev1
+
+sudo /usr/share/openvswitch/scripts/ovs-ctl start
+
+# Build Test Container
+sudo docker build ./testing/docker/ci_test_device1 -t ci_test_device1 -f  ./testing/docker/ci_test_device1/Dockerfile
+
+cat <<EOF >local/system.json
+{
+  "network": {
+    "device_intf": "endev0a",
+    "internet_intf": "eth0"
+  },
+  "log_level": "DEBUG",
+  "monitor_period": 30
+}
+EOF
+
+sudo cmd/install
+
+TESTERS=$(jq -r 'keys[]' $MATRIX)
+for tester in $TESTERS; do
+  testrun_log=$TEST_DIR/${tester}_testrun.log
+  device_log=$TEST_DIR/${tester}_device.log
+
+  image=$(jq -r .$tester.image $MATRIX)
+  ethmac=$(jq -r .$tester.ethmac $MATRIX)
+  args=$(jq -r .$tester.args $MATRIX)
+
+  touch $testrun_log
+  sudo timeout 900 cmd/start --single-intf > $testrun_log 2>&1 &
+  TPID=$!
+
+  # Time to wait for testrun to be ready
+  WAITING=600
+  for i in `seq 1 $WAITING`; do
+    tail -1 $testrun_log
+      if [[ -n $(fgrep "Waiting for devices on the network" $testrun_log) ]]; then
+          break
+      fi
+
+      if [[ ! -d /proc/$TPID ]]; then
+          cat $testrun_log
+          echo "error encountered starting test run"
+          exit 1
+      fi
+    
+      sleep 1
+  done
+
+  if [[ $i -eq $WAITING ]]; then
+      cat $testrun_log
+      echo "failed after waiting $WAITING seconds for test-run start"
+      exit 1
+  fi
+
+  # Load Test Container
+  sudo docker run -d \
+    --network=endev1 \
+    --mac-address=$ethmac \
+    --cap-add=NET_ADMIN \
+    -v /tmp:/out \
+    --privileged \
+    --name=$tester \
+    ci_test_device1 $args
+  
+  wait $TPID
+  # Following line indicates that tests are completed but wait till it exits
+  # Completed running test modules on device with mac addr 7e:41:12:d2:35:6a
+  #Change this line! - LOGGER.info(f"""Completed running test modules on device
+  #          with mac addr {device.mac_addr}""")
+
+  ls runtime
+  more runtime/network/*.log
+  sudo docker kill $tester
+  sudo docker logs $tester | cat 
+  
+  cp runtime/test/${ethmac//:/}/results.json $TEST_DIR/$tester.json
+  more $TEST_DIR/$tester.json
+  more $testrun_log
+
+done
+
+pytest -s testing/test_tests.py
+
+exit $?
diff --git a/testing/test_tests.json b/testing/test_tests.json
new file mode 100644
index 000000000..076e9149e
--- /dev/null
+++ b/testing/test_tests.json
@@ -0,0 +1,19 @@
+{
+    "tester1": {
+        "image": "test-run/ci_test1",
+        "args": "oddservices",
+        "ethmac": "02:42:aa:00:00:01",
+        "expected_results": {
+            "security.nmap.ports": "non-compliant"
+        }
+    },
+    "tester2": {
+        "image": "test-run/ci_test1",
+        "args": "",
+        "ethmac": "02:42:aa:00:00:02",
+        "expected_results": {
+            "security.nmap.ports": "compliant"
+        }
+    }
+    
+}
\ No newline at end of file
diff --git a/testing/test_tests.py b/testing/test_tests.py
new file mode 100644
index 000000000..7c60484f0
--- /dev/null
+++ b/testing/test_tests.py
@@ -0,0 +1,102 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Test assertions for CI testing of tests """
+# Temporarily disabled because using Pytest fixtures
+# TODO refactor fixtures to not trigger error
+# pylint: disable=redefined-outer-name
+
+import json
+import pytest
+import os
+import glob
+import itertools
+
+from pathlib import Path
+from dataclasses import dataclass
+
+TEST_MATRIX = 'test_tests.json'
+RESULTS_PATH = '/tmp/results/*.json'
+
+@dataclass(frozen=True)
+class TestResult:
+  name: str
+  result: str
+  __test__ = False
+
+
+def collect_expected_results(expected_results):
+  """ Yields results from expected_results property of the test matrix"""
+  for name, result in expected_results.items():
+    yield TestResult(name, result)
+
+
+def collect_actual_results(results_dict):
+  """ Yields results from an already loaded testrun results file """
+  # "module"."results".[list]."result"
+  for maybe_module, child in results_dict.items():
+    if 'results' in child and maybe_module != 'baseline':
+      for test in child['results']:
+        yield TestResult(test['name'], test['result'])
+
+
+@pytest.fixture
+def test_matrix():
+  basedir = os.path.dirname(os.path.abspath(__file__))
+  with open(os.path.join(basedir, TEST_MATRIX), encoding='utf-8') as f:
+    return json.load(f)
+
+
+@pytest.fixture
+def results():
+  results = {}
+  for file in [Path(x) for x in glob.glob(RESULTS_PATH)]:
+    with open(file, encoding='utf-8') as f:
+      results[file.stem] = json.load(f)
+  return results
+
+
+def test_tests(results, test_matrix):
+  """ Check if each testers expect results were obtained """
+  for tester, props in test_matrix.items():
+    expected = set(collect_expected_results(props['expected_results']))
+    actual = set(collect_actual_results(results[tester]))
+
+    assert expected.issubset(actual), f'{tester} expected results not obtained'
+
+def test_list_tests(capsys, results, test_matrix):
+  all_tests = set(itertools.chain.from_iterable(
+      [collect_actual_results(results[x]) for x in results.keys()]))
+
+  ci_pass = set([test 
+    for testers in test_matrix.values() 
+    for test, result in testers['expected_results'].items() 
+    if result == 'compliant'])
+
+  ci_fail = set([test 
+    for testers in test_matrix.values() 
+    for test, result in testers['expected_results'].items() 
+    if result == 'non-compliant'])
+
+  with capsys.disabled():
+    print('============')
+    print('============')
+    print('tests seen:')
+    print('\n'.join([x.name for x in all_tests]))
+    print('\ntesting for pass:')
+    print('\n'.join(ci_pass))
+    print('\ntesting for pass:')
+    print('\n'.join(ci_pass))
+
+  assert True