From 21ce171c786345cb9d6b2c6c9d03047310d21313 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Wed, 31 Jul 2024 09:50:15 +0100 Subject: [PATCH 01/30] changed the tests order in test_api.py --- testing/api/test_api.py | 498 +++++++++++++++++++++------------------- 1 file changed, 257 insertions(+), 241 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 33163cecc..00961d72f 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -224,25 +224,85 @@ def local_get_devices(): ) ) +# Tests for system endpoints def test_get_system_interfaces(testrun): # pylint: disable=W0613 """Tests API system interfaces against actual local interfaces""" + # Send a GET request to the API to retrieve system interfaces r = requests.get(f"{API}/system/interfaces", timeout=5) + # Parse the JSON response response = r.json() + # Retrieve the actual network interfaces local_interfaces = get_network_interfaces() - assert set(response.keys()) == set(local_interfaces) - # schema expects a flat list + # Check if status code is 200 (OK) + assert r.status_code == 200, f"status code is {r.status_code}" + # Check if the key are in the response + assert set(response.keys()) == set(local_interfaces) + # Ensure that all values in the response are strings assert all(isinstance(x, str) for x in response) +def test_update_system_config(testrun): # pylint: disable=W0613 + """Test update system configuration endpoint ('/system/config')""" + pass -def test_status_idle(testrun): # pylint: disable=W0613 - until_true( - lambda: query_system_status().lower() == "idle", - "system status is `idle`", - 30, +def test_get_system_config(testrun): # pylint: disable=W0613 + """Tests get system configuration endpoint ('/system/config')""" + # Send a GET request to the API to retrieve system configuration + r = requests.get(f"{API}/system/config", timeout=5) + + # Open the local system configuration file + with open( + SYSTEM_CONFIG_PATH, + encoding="utf-8" + ) as f: + local_config = json.load(f) + + # Parse the JSON response + api_config = r.json() + + # Check if status code is 200 (OK) + assert r.status_code == 200, f"status code is {r.status_code}" + + # Validate structure + assert set(dict_paths(api_config)) | set(dict_paths(local_config)) == set( + dict_paths(api_config) + ) + + # Check if the device interface in the local config matches the API config + assert ( + local_config["network"]["device_intf"] + == api_config["network"]["device_intf"] + ) + + # Check if the internet interface in the local config matches the API config + assert ( + local_config["network"]["internet_intf"] + == api_config["network"]["internet_intf"] ) +def test_start_testrun_started_successfully( + testing_devices, # pylint: disable=W0613 + testrun): # pylint: disable=W0613 + """Test if testrun started successfully """ + + # Payload with device details + payload = {"device": { + "mac_addr": BASELINE_MAC_ADDR, + "firmware": "asd", + "test_modules": { + "dns": {"enabled": False}, + "connection": {"enabled": True}, + "ntp": {"enabled": False}, + "baseline": {"enabled": False}, + "nmap": {"enabled": False} + }}} + # Send the post request + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + # Check if the response status code is 200 (OK) + assert r.status_code == 200 + # Currently not working due to blocking during monitoring period @pytest.mark.skip() def test_status_in_progress(testing_devices, testrun): # pylint: disable=W0613 @@ -265,6 +325,196 @@ def test_status_in_progress(testing_devices, testrun): # pylint: disable=W0613 600, ) +# Currently not working due to blocking during monitoring period +@pytest.mark.skip() +def test_start_testrun_already_in_progress( + testing_devices, # pylint: disable=W0613 + testrun): # pylint: disable=W0613 + payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + until_true( + lambda: query_system_status().lower() == "waiting for device", + "system status is `waiting for device`", + 30, + ) + + start_test_device("x123", BASELINE_MAC_ADDR) + + until_true( + lambda: query_system_status().lower() == "in progress", + "system status is `in progress`", + 600, + ) + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + assert r.status_code == 409 + +@pytest.mark.skip() +def test_trigger_run(testing_devices, testrun): # pylint: disable=W0613 + payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + assert r.status_code == 200 + + until_true( + lambda: query_system_status().lower() == "waiting for device", + "system status is `waiting for device`", + 30, + ) + + start_test_device("x123", BASELINE_MAC_ADDR) + + until_true( + lambda: query_system_status().lower() == "compliant", + "system status is `complete`", + 600, + ) + + stop_test_device("x123") + + # Validate response + r = requests.get(f"{API}/system/status", timeout=5) + response = r.json() + pretty_print(response) + + # Validate results + results = {x["name"]: x for x in response["tests"]["results"]} + print(results) + # there are only 3 baseline tests + assert len(results) == 3 + + # Validate structure + with open( + os.path.join( + os.path.dirname(__file__), "mockito/running_system_status.json" + ), encoding="utf-8" + ) as f: + mockito = json.load(f) + + # validate structure + assert set(dict_paths(mockito)).issubset(set(dict_paths(response))) + + # Validate results structure + assert set(dict_paths(mockito["tests"]["results"][0])).issubset( + set(dict_paths(response["tests"]["results"][0])) + ) + + # Validate a result + assert results["baseline.compliant"]["result"] == "Compliant" + +@pytest.mark.skip() +def test_stop_running_test(testing_devices, testrun): # pylint: disable=W0613 + payload = {"device": {"mac_addr": ALL_MAC_ADDR, "firmware": "asd"}} + r = requests.post(f"{API}/system/start", data=json.dumps(payload), + timeout=10) + assert r.status_code == 200 + + until_true( + lambda: query_system_status().lower() == "waiting for device", + "system status is `waiting for device`", + 30, + ) + + start_test_device("x12345", ALL_MAC_ADDR) + + until_true( + lambda: query_test_count() > 1, + "system status is `complete`", + 1000, + ) + + stop_test_device("x12345") + + # Validate response + r = requests.post(f"{API}/system/stop", timeout=5) + response = r.json() + pretty_print(response) + assert response == {"success": "Testrun stopped"} + time.sleep(1) + + # Validate response + r = requests.get(f"{API}/system/status", timeout=5) + response = r.json() + pretty_print(response) + + assert response["status"] == "Cancelled" + +def test_stop_running_not_running(testrun): # pylint: disable=W0613 + # Validate response + r = requests.post(f"{API}/system/stop", + timeout=10) + response = r.json() + pretty_print(response) + + assert r.status_code == 404 + assert response["error"] == "Testrun is not currently running" + +@pytest.mark.skip() +def test_multiple_runs(testing_devices, testrun): # pylint: disable=W0613 + payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} + r = requests.post(f"{API}/system/start", data=json.dumps(payload), + timeout=10) + assert r.status_code == 200 + print(r.text) + + until_true( + lambda: query_system_status().lower() == "waiting for device", + "system status is `waiting for device`", + 30, + ) + + start_test_device("x123", BASELINE_MAC_ADDR) + + until_true( + lambda: query_system_status().lower() == "compliant", + "system status is `complete`", + 900, + ) + + stop_test_device("x123") + + # Validate response + r = requests.get(f"{API}/system/status", timeout=5) + response = r.json() + pretty_print(response) + + # Validate results + results = {x["name"]: x for x in response["tests"]["results"]} + print(results) + # there are only 3 baseline tests + assert len(results) == 3 + + payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} + r = requests.post(f"{API}/system/start", data=json.dumps(payload), + timeout=10) + # assert r.status_code == 200 + # returns 409 + print(r.text) + + until_true( + lambda: query_system_status().lower() == "waiting for device", + "system status is `waiting for device`", + 30, + ) + + start_test_device("x123", BASELINE_MAC_ADDR) + + until_true( + lambda: query_system_status().lower() == "compliant", + "system status is `complete`", + 900, + ) + + stop_test_device("x123") + +def test_status_idle(testrun): # pylint: disable=W0613 + """Test system status 'idle' endpoint (/system/status)""" + until_true( + lambda: query_system_status().lower() == "idle", + "system status is `idle`", + 30, + ) + +# Tests for device endpoints @pytest.mark.skip() def test_status_non_compliant(testing_devices, testrun): # pylint: disable=W0613 @@ -298,7 +548,6 @@ def test_status_non_compliant(testing_devices, testrun): # pylint: disable=W0613 stop_test_device("x123") - def test_create_get_devices(empty_devices_dir, testrun): # pylint: disable=W0613 device_1 = { "manufacturer": "Google", @@ -361,7 +610,6 @@ def test_create_get_devices(empty_devices_dir, testrun): # pylint: disable=W0613 [device_1[key], device_2[key]] ) - def test_delete_device_success(empty_devices_dir, testrun): # pylint: disable=W0613 device_1 = { "manufacturer": "Google", @@ -439,7 +687,6 @@ def test_delete_device_success(empty_devices_dir, testrun): # pylint: disable=W0 [device_2[key]] ) - def test_delete_device_not_found(empty_devices_dir, testrun): # pylint: disable=W0613 device_1 = { "manufacturer": "Google", @@ -478,7 +725,6 @@ def test_delete_device_not_found(empty_devices_dir, testrun): # pylint: disable= assert r.status_code == 404 assert len(local_get_devices()) == 0 - def test_delete_device_no_mac(empty_devices_dir, testrun): # pylint: disable=W0613 device_1 = { "manufacturer": "Google", @@ -512,7 +758,6 @@ def test_delete_device_no_mac(empty_devices_dir, testrun): # pylint: disable=W06 assert r.status_code == 400 assert len(local_get_devices()) == 1 - # Currently not working due to blocking during monitoring period @pytest.mark.skip() def test_delete_device_testrun_running(testing_devices, testrun): # pylint: disable=W0613 @@ -552,49 +797,6 @@ def test_delete_device_testrun_running(testing_devices, testrun): # pylint: disa timeout=5) assert r.status_code == 403 - -def test_start_testrun_started_successfully( - testing_devices, # pylint: disable=W0613 - testrun): # pylint: disable=W0613 - payload = {"device": { - "mac_addr": BASELINE_MAC_ADDR, - "firmware": "asd", - "test_modules": { - "dns": {"enabled": False}, - "connection": {"enabled": True}, - "ntp": {"enabled": False}, - "baseline": {"enabled": False}, - "nmap": {"enabled": False} - }}} - r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) - assert r.status_code == 200 - - -# Currently not working due to blocking during monitoring period -@pytest.mark.skip() -def test_start_testrun_already_in_progress( - testing_devices, # pylint: disable=W0613 - testrun): # pylint: disable=W0613 - payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} - r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) - - until_true( - lambda: query_system_status().lower() == "waiting for device", - "system status is `waiting for device`", - 30, - ) - - start_test_device("x123", BASELINE_MAC_ADDR) - - until_true( - lambda: query_system_status().lower() == "in progress", - "system status is `in progress`", - 600, - ) - r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) - assert r.status_code == 409 - - def test_start_system_not_configured_correctly( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -924,32 +1126,6 @@ def test_system_latest_version(testrun): # pylint: disable=W0613 assert updated_system_version is False -def test_get_system_config(testrun): # pylint: disable=W0613 - r = requests.get(f"{API}/system/config", timeout=5) - - with open( - SYSTEM_CONFIG_PATH, - encoding="utf-8" - ) as f: - local_config = json.load(f) - - api_config = r.json() - - # validate structure - assert set(dict_paths(api_config)) | set(dict_paths(local_config)) == set( - dict_paths(api_config) - ) - - assert ( - local_config["network"]["device_intf"] - == api_config["network"]["device_intf"] - ) - assert ( - local_config["network"]["internet_intf"] - == api_config["network"]["internet_intf"] - ) - - def test_invalid_path_get(testrun): # pylint: disable=W0613 r = requests.get(f"{API}/blah/blah", timeout=5) response = r.json() @@ -964,166 +1140,6 @@ def test_invalid_path_get(testrun): # pylint: disable=W0613 assert set(dict_paths(mockito)) == set(dict_paths(response)) -@pytest.mark.skip() -def test_trigger_run(testing_devices, testrun): # pylint: disable=W0613 - payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} - r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) - assert r.status_code == 200 - - until_true( - lambda: query_system_status().lower() == "waiting for device", - "system status is `waiting for device`", - 30, - ) - - start_test_device("x123", BASELINE_MAC_ADDR) - - until_true( - lambda: query_system_status().lower() == "compliant", - "system status is `complete`", - 600, - ) - - stop_test_device("x123") - - # Validate response - r = requests.get(f"{API}/system/status", timeout=5) - response = r.json() - pretty_print(response) - - # Validate results - results = {x["name"]: x for x in response["tests"]["results"]} - print(results) - # there are only 3 baseline tests - assert len(results) == 3 - - # Validate structure - with open( - os.path.join( - os.path.dirname(__file__), "mockito/running_system_status.json" - ), encoding="utf-8" - ) as f: - mockito = json.load(f) - - # validate structure - assert set(dict_paths(mockito)).issubset(set(dict_paths(response))) - - # Validate results structure - assert set(dict_paths(mockito["tests"]["results"][0])).issubset( - set(dict_paths(response["tests"]["results"][0])) - ) - - # Validate a result - assert results["baseline.compliant"]["result"] == "Compliant" - - -@pytest.mark.skip() -def test_stop_running_test(testing_devices, testrun): # pylint: disable=W0613 - payload = {"device": {"mac_addr": ALL_MAC_ADDR, "firmware": "asd"}} - r = requests.post(f"{API}/system/start", data=json.dumps(payload), - timeout=10) - assert r.status_code == 200 - - until_true( - lambda: query_system_status().lower() == "waiting for device", - "system status is `waiting for device`", - 30, - ) - - start_test_device("x12345", ALL_MAC_ADDR) - - until_true( - lambda: query_test_count() > 1, - "system status is `complete`", - 1000, - ) - - stop_test_device("x12345") - - # Validate response - r = requests.post(f"{API}/system/stop", timeout=5) - response = r.json() - pretty_print(response) - assert response == {"success": "Testrun stopped"} - time.sleep(1) - - # Validate response - r = requests.get(f"{API}/system/status", timeout=5) - response = r.json() - pretty_print(response) - - assert response["status"] == "Cancelled" - - -def test_stop_running_not_running(testrun): # pylint: disable=W0613 - # Validate response - r = requests.post(f"{API}/system/stop", - timeout=10) - response = r.json() - pretty_print(response) - - assert r.status_code == 404 - assert response["error"] == "Testrun is not currently running" - -@pytest.mark.skip() -def test_multiple_runs(testing_devices, testrun): # pylint: disable=W0613 - payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} - r = requests.post(f"{API}/system/start", data=json.dumps(payload), - timeout=10) - assert r.status_code == 200 - print(r.text) - - until_true( - lambda: query_system_status().lower() == "waiting for device", - "system status is `waiting for device`", - 30, - ) - - start_test_device("x123", BASELINE_MAC_ADDR) - - until_true( - lambda: query_system_status().lower() == "compliant", - "system status is `complete`", - 900, - ) - - stop_test_device("x123") - - # Validate response - r = requests.get(f"{API}/system/status", timeout=5) - response = r.json() - pretty_print(response) - - # Validate results - results = {x["name"]: x for x in response["tests"]["results"]} - print(results) - # there are only 3 baseline tests - assert len(results) == 3 - - payload = {"device": {"mac_addr": BASELINE_MAC_ADDR, "firmware": "asd"}} - r = requests.post(f"{API}/system/start", data=json.dumps(payload), - timeout=10) - # assert r.status_code == 200 - # returns 409 - print(r.text) - - until_true( - lambda: query_system_status().lower() == "waiting for device", - "system status is `waiting for device`", - 30, - ) - - start_test_device("x123", BASELINE_MAC_ADDR) - - until_true( - lambda: query_system_status().lower() == "compliant", - "system status is `complete`", - 900, - ) - - stop_test_device("x123") - - def test_create_invalid_chars(empty_devices_dir, testrun): # pylint: disable=W0613 # local_delete_devices(ALL_DEVICES) # We must start test run with no devices in local/devices for this test From d57b39940051a1a59316eae9d9610a8856f4910e Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Wed, 31 Jul 2024 14:11:40 +0100 Subject: [PATCH 02/30] added tests for '/system/config' POST endpoint --- testing/api/test_api.py | 72 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 00961d72f..463da40f6 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -38,6 +38,7 @@ TESTING_DEVICES = "../device_configs" PROFILES_DIRECTORY = "local/risk_profiles" SYSTEM_CONFIG_PATH = "local/system.json" +SYSTEM_CONFIG_RESTORE_PATH = "testing/api/system.json" BASELINE_MAC_ADDR = "02:42:aa:00:01:01" ALL_MAC_ADDR = "02:42:aa:00:00:01" @@ -226,9 +227,18 @@ def local_get_devices(): # Tests for system endpoints +@pytest.fixture() +def restore_config(): + """Restore the original configuration (system.json) after the test""" + yield + # Restore system.json from 'testing/api/' after the test + if os.path.exists(SYSTEM_CONFIG_RESTORE_PATH): + shutil.copy(SYSTEM_CONFIG_RESTORE_PATH, SYSTEM_CONFIG_PATH) + def test_get_system_interfaces(testrun): # pylint: disable=W0613 """Tests API system interfaces against actual local interfaces""" - # Send a GET request to the API to retrieve system interfaces + + # Send a GET request to the API to retrieve system interfaces r = requests.get(f"{API}/system/interfaces", timeout=5) # Parse the JSON response response = r.json() @@ -242,9 +252,65 @@ def test_get_system_interfaces(testrun): # pylint: disable=W0613 # Ensure that all values in the response are strings assert all(isinstance(x, str) for x in response) -def test_update_system_config(testrun): # pylint: disable=W0613 +def test_update_system_config(testrun, restore_config): # pylint: disable=W0613 """Test update system configuration endpoint ('/system/config')""" - pass + + # Configuration data to update + updated_system_config = { + "network": { + "device_intf": "updated_endev0a", + "internet_intf": "updated_wlan1" + }, + "log_level": "DEBUG" + } + + # Send the post request to update the system configuration + r = requests.post(f"{API}/system/config", + data=json.dumps(updated_system_config), + timeout=5) + + # Check if status code is 200 (OK) + assert r.status_code == 200 + + # Parse the JSON response + response = r.json() + + # Check if the response["network"]["device_intf"] has been updated + assert ( + response["network"]["device_intf"] + == updated_system_config["network"]["device_intf"] + ) + + # Check if the response["network"]["internet_intf"] has been updated + assert ( + response["network"]["internet_intf"] + == updated_system_config["network"]["internet_intf"] + ) + + # Check if the response["log_level"] has been updated + assert ( + response["log_level"] + == updated_system_config["log_level"] + ) + +def test_update_system_config_invalid_config(testrun, restore_config): # pylint: disable=W0613 + """Test invalid configuration file for update system configuration""" + + # Configuration data to update + updated_system_config = { + "network": { + "device_intf": "updated_endev0a", + "internet_intf": "updated_wlan1" + } + } + + # Send the post request to update the system configuration + r = requests.post(f"{API}/system/config", + data=json.dumps(updated_system_config), + timeout=5) + + # Check if status code is 500 (bad request) + assert r.status_code == 500 def test_get_system_config(testrun): # pylint: disable=W0613 """Tests get system configuration endpoint ('/system/config')""" From 5ac2a0c3ac164dc55f9c13f0fa6b5ecc57001499 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Wed, 31 Jul 2024 15:12:32 +0100 Subject: [PATCH 03/30] added the tests for 'system/shutdown' endpoint --- testing/api/test_api.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 463da40f6..39bcaa366 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -309,7 +309,7 @@ def test_update_system_config_invalid_config(testrun, restore_config): # pylint: data=json.dumps(updated_system_config), timeout=5) - # Check if status code is 500 (bad request) + # Check if status code is 500 (Invalid config) assert r.status_code == 500 def test_get_system_config(testrun): # pylint: disable=W0613 @@ -580,6 +580,42 @@ def test_status_idle(testrun): # pylint: disable=W0613 30, ) +def test_system_shutdown(testrun): # pylint: disable=W0613 + """Test the shutdown system endpoint""" + # Send a POST request to initiate the system shutdown + r = requests.post(f"{API}/system/shutdown", timeout=5) + + # Check if the response status code is 200 (OK) + assert r.status_code == 200, f"Expected status code 200, got {r.status_code}" + +def test_system_shutdown_in_progress(testrun): # pylint: disable=W0613 + """Test system shutdown during an in-progress test""" + # Payload with device details to start a test + payload = { + "device": { + "mac_addr": BASELINE_MAC_ADDR, + "firmware": "asd", + "test_modules": { + "dns": {"enabled": False}, + "connection": {"enabled": True}, + "ntp": {"enabled": False}, + "baseline": {"enabled": False}, + "nmap": {"enabled": False} + } + } + } + # Start a test run + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + # Check if the response status code is 200 (OK) for starting the test + assert r.status_code == 200 + + # Attempt to shutdown while the test is running + r = requests.post(f"{API}/system/shutdown", timeout=5) + + # Check if the response status code is 400 (test in progress) + assert r.status_code == 400 + # Tests for device endpoints @pytest.mark.skip() From ebdc13f424c56aef6af19394655af97cf0ae5ad1 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Wed, 31 Jul 2024 16:42:29 +0100 Subject: [PATCH 04/30] added the test for GET '/reports' endpoint, updated 'test_update_system_config_invalid_config' to return error 400 --- testing/api/test_api.py | 47 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 39bcaa366..c137b92eb 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -309,8 +309,8 @@ def test_update_system_config_invalid_config(testrun, restore_config): # pylint: data=json.dumps(updated_system_config), timeout=5) - # Check if status code is 500 (Invalid config) - assert r.status_code == 500 + # Check if status code is 400 (Invalid config) + assert r.status_code == 400 def test_get_system_config(testrun): # pylint: disable=W0613 """Tests get system configuration endpoint ('/system/config')""" @@ -604,11 +604,12 @@ def test_system_shutdown_in_progress(testrun): # pylint: disable=W0613 } } } - # Start a test run + # Start a test r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) - # Check if the response status code is 200 (OK) for starting the test - assert r.status_code == 200 + # Check if status code is not 200 (OK) + if r.status_code != 200: + raise ValueError(f"Api request failed with code: {r.status_code}") # Attempt to shutdown while the test is running r = requests.post(f"{API}/system/shutdown", timeout=5) @@ -616,6 +617,42 @@ def test_system_shutdown_in_progress(testrun): # pylint: disable=W0613 # Check if the response status code is 400 (test in progress) assert r.status_code == 400 +# Tests for reports endpoints + +def test_get_reports(testrun): # pylint: disable=W0613 + """Test for get reports endpoint""" + # Send a GET request to the /reports endpoint + r = requests.get(f"{API}/reports", timeout=5) + + # Check if the status code is 200 (OK) + assert r.status_code == 200 + + # Parse the JSON response + response = r.json() + + # Check if the response is a list + assert isinstance(response, list) + + # Iterate through each report + for report in response: + # Check if the report is dict + assert isinstance(report, dict) + # Check if "mac_adrr" key is in report + assert "mac_addr" in report + # Check if "device" key is in report + assert "device" in report + # Check if "status" key is in report + assert "status" in report + # Check if "started" key is in report + assert "started" in report + # Check if "finished" key is in report + assert "finished" in report + # Check if "tests" key is in report + assert "tests" in report + # Check if "report" key is in report + assert "report" in report + # Check if "device" key is in report + # Tests for device endpoints @pytest.mark.skip() From b1c1ecc2be674744791fa2eea7e2da1ca85832e6 Mon Sep 17 00:00:00 2001 From: J Boddey Date: Wed, 31 Jul 2024 20:46:42 +0100 Subject: [PATCH 05/30] Check for missing fields Signed-off-by: J Boddey --- framework/python/src/api/api.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 719710694..0ffc48062 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -169,6 +169,17 @@ async def post_sys_config(self, request: Request, response: Response): config = (await request.body()).decode("UTF-8") config_json = json.loads(config) self._session.set_config(config_json) + + # Validate req fields + if ("network" not in config_json or + "device_intf" not in config_json.get("network") or + "internet_intf" not in config_json.get("network") or + "log_level" not in config_json): + response.status_code = status.HTTP_400_BAD_REQUEST + return self._generate_msg( + False, + "Configuration is missing required fields") + # Catch JSON Decode error etc except JSONDecodeError: response.status_code = status.HTTP_400_BAD_REQUEST From 413f59c8eb44354df6b9946e448733876bf30ff3 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Thu, 1 Aug 2024 14:37:21 +0100 Subject: [PATCH 06/30] added tests for delete profile (404, 400), added tests for create and update profile (400), added test 'run_test_and_get_report' skipped due to blocking during testing phase --- testing/api/test_api.py | 175 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index c137b92eb..88f0d0430 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -651,7 +651,75 @@ def test_get_reports(testrun): # pylint: disable=W0613 assert "tests" in report # Check if "report" key is in report assert "report" in report - # Check if "device" key is in report + +def test_get_reports_no_reports(testrun): # pylint: disable=W0613 + """Test get reports when no reports exist.""" + r = requests.get(f"{API}/reports", timeout=5) + assert r.status_code == 200, f"Expected 200, got {r.status_code}" + response = r.json() + assert response == [], "Expected empty list when no reports exist" + +@pytest.mark.skip() +def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 + """Initiate a test run, ensure report generation, and fetch the report.""" + # Step 1: Prepare and start the test run + payload = { + "device": { + "mac_addr": BASELINE_MAC_ADDR, + "firmware": "asd", + "test_modules": { + "dns": {"enabled": False}, + "connection": {"enabled": True}, + "ntp": {"enabled": False}, + "baseline": {"enabled": False}, + "nmap": {"enabled": False} + } + } + } + + # Send the post request to start the test + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + assert r.status_code == 200 + + # Step 2: Start the device for the test + device_name = "test_device" + start_test_device(device_name, BASELINE_MAC_ADDR) + + # Step 3: Wait for the test to complete + max_retries = 300 # time limit + for _ in range(max_retries): + time.sleep(1) + status = query_system_status().lower() + print(f"Current system status: {status}") # Additional logging + if status in ["compliant", "non-compliant", "cancelled"]: + break + else: + # Timeout reached without a valid end state + stop_test_device(device_name) + final_status = query_system_status().lower() # Get the final status + print("Final system status:", final_status) # Log the final status + pytest.fail("Test run did not complete within the expected time. Final status: " + final_status) + + # Step 4: Fetch the generated report + r = requests.get(f"{API}/report/{BASELINE_MAC_ADDR}", timeout=5) + assert r.status_code == 200, "Failed to fetch the report" + report_data = r.json() + print(f"Reports are {report_data}") + + # Step 5: Stop the test device + stop_test_device(device_name) + +def test_delete_report(testrun): # pylint: disable=W0613 + """Test the delete report endpoint""" + pass + +def test_get_report(testrun): # pylint: disable=W0613 + """Test the get report endpoint""" + pass + +def test_export_report(testrun): # pylint: disable=W0613 + """Test the export report endpoint""" + pass # Tests for device endpoints @@ -1563,6 +1631,62 @@ def test_update_profile(testrun, reset_profiles, add_profile): # pylint: disable # Check if profile was updated assert updated_profile_check is not None +def test_update_profile_invalid_json(testrun, reset_profiles, add_profile): # pylint: disable=W0613 + """Test for update profile invalid JSON payload (no 'name')""" + # Load the new profile using add_profile fixture + add_profile("new_profile.json") + # Load the updated profile using load_profile utility method + updated_profile = load_profile("profile_invalid_format.json") + + # Send the post request to update the profile + r = requests.post( + f"{API}/profiles", + data=json.dumps(updated_profile), + timeout=5) + + # Check if status code is 400 (Bad request) + assert r.status_code == 400 + # Parse the response + response = r.json() + # Check if "error" key in response + assert "error" in response + +def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable=W0613 + """Test for create profile invalid JSON payload """ + + # Load the profile using load_profile utility method + new_profile = load_profile("profile_invalid_format.json") + + # Send the post request to update the profile + r = requests.post( + f"{API}/profiles", + data=json.dumps(new_profile), + timeout=5) + + # Check if status code is 400 (Bad request) + assert r.status_code == 400 + # Parse the response + response = r.json() + # Check if "error" key in response + assert "error" in response + + # Load the 2nd profile + new_profile_2 = {"name": "New Profile"} + + # Send the post request to update the profile + r = requests.post( + f"{API}/profiles", + data=json.dumps(new_profile_2), + timeout=5) + + # Check if status code is 400 (Bad request) + assert r.status_code == 400 + # Parse the response + response = r.json() + # Check if "error" key in response + assert "error" in response + + def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for delete profile""" @@ -1599,3 +1723,52 @@ def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable ) # Check if profile was deleted assert deleted_profile is None + +def test_delete_profile_no_profile(testrun, reset_profiles): # pylint: disable=W0613 + """Test delete profile if the profile does not exists""" + # Load the profile to delete + profile_to_delete = load_profile("new_profile.json") + + # Delete the profile + r = requests.delete( + f"{API}/profiles", + data=json.dumps(profile_to_delete), + timeout=5) + + # Check if status code is 404 (Profile does not exist) + assert r.status_code == 404 + +def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable=W0613 + """Test for delete profile wrong JSON payload""" + + profile_to_delete = {} #load_profile("profile_invalid_format.json") + + # Delete the profile + r = requests.delete( + f"{API}/profiles", + data=json.dumps(profile_to_delete), + timeout=5) + + # Parse the response + response = r.json() + + # Check if status code is 400 (bad request) + assert r.status_code == 400 + # Check if "error" key in response + assert "error" in response + + profile_to_delete_2 = load_profile("profile_invalid_format.json") + + # Delete the profile + r = requests.delete( + f"{API}/profiles", + data=json.dumps(profile_to_delete_2), + timeout=5) + + # Parse the response + response = r.json() + + # Check if status code is 400 (bad request) + assert r.status_code == 400 + # Check if "error" key in response + assert "error" in response From c5275cd928b27e73146407558a66561c5234df9b Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Thu, 1 Aug 2024 14:38:48 +0100 Subject: [PATCH 07/30] added a new json file in '/testing/api/' used in 400 error tests --- .../api/profiles/profile_invalid_format.json | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 testing/api/profiles/profile_invalid_format.json diff --git a/testing/api/profiles/profile_invalid_format.json b/testing/api/profiles/profile_invalid_format.json new file mode 100644 index 000000000..a88e8e6b7 --- /dev/null +++ b/testing/api/profiles/profile_invalid_format.json @@ -0,0 +1,144 @@ +{ + "status": "Draft", + "created": "2024-05-23 12:38:26", + "version": "v1.3", + "questions": [ + { + "question": "What type of device is this?", + "type": "select", + "options": [ + "IoT Sensor", + "IoT Controller", + "Smart Device", + "Something else" + ], + "answer": "IoT Sensor", + "validation": { + "required": true + } + }, + { + "question": "How will this device be used at Google?", + "type": "text-long", + "answer": "Installed in a building", + "validation": { + "max": "128", + "required": true + } + }, + { + "question": "What is the email of the device owner(s)?", + "type": "email-multiple", + "answer": "boddey@google.com, cmeredith@google.com", + "validation": { + "required": true, + "max": "128" + } + }, + { + "question": "Is this device going to be managed by Google or a third party?", + "type": "select", + "options": [ + "Google", + "Third Party" + ], + "answer": "Google", + "validation": { + "required": true + } + }, + { + "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", + "type": "select", + "options": [ + "Yes", + "No", + "N/A" + ], + "default": "N/A", + "answer": "Yes", + "validation": { + "required": true + } + }, + { + "question": "Are any of the following statements true about your device?", + "description": "This tells us about the data your device will collect", + "type": "select-multiple", + "answer": [ + 0, + 2 + ], + "options": [ + "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", + "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", + "The device stream confidential business data in real-time (seconds)?" + ] + }, + { + "question": "Which of the following statements are true about this device?", + "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", + "type": "select-multiple", + "answer": [ + 0, + 1, + 5 + ], + "options": [ + "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", + "Data transmission occurs across less-trusted networks (e.g. the internet).", + "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", + "A confidentiality breach during transmission would have a substantial negative impact", + "The device encrypts data during transmission", + "The device network protocol is well-established and currently used by Google" + ] + }, + { + "question": "Does the network protocol assure server-to-client identity verification?", + "type": "select", + "answer": "Yes", + "options": [ + "Yes", + "No", + "I don't know" + ], + "validation": { + "required": true + } + }, + { + "question": "Click the statements that best describe the characteristics of this device.", + "description": "This tells us about how this device is managed remotely.", + "type": "select-multiple", + "answer": [ + 0, + 1, + 2 + ], + "options": [ + "PII/PHI, or confidential business data is accessible from the device without authentication", + "Unrecoverable actions (e.g. disk wipe) can be performed remotely", + "Authentication is required for remote access", + "The management interface is accessible from the public internet", + "Static credentials are used for administration" + ] + }, + { + "question": "Are any of the following statements true about this device?", + "description": "This informs us about what other systems and processes this device is a part of.", + "type": "select-multiple", + "answer": [ + 2, + 3 + ], + "options": [ + "The device monitors an environment for active risks to human life.", + "The device is used to convey people, or critical property.", + "The device controls robotics in human-accessible spaces.", + "The device controls physical access systems.", + "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", + "The device's failure would cause faults in other high-criticality processes." + ] + } + ] +} \ No newline at end of file From 2beff3e2104edbf20b2482c0660ef809d6b28c5d Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Thu, 1 Aug 2024 14:41:15 +0100 Subject: [PATCH 08/30] added error handling if 'name' and 'questions' not in profile json --- framework/python/src/api/api.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 0ffc48062..f6c44b2fe 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -168,9 +168,8 @@ async def post_sys_config(self, request: Request, response: Response): try: config = (await request.body()).decode("UTF-8") config_json = json.loads(config) - self._session.set_config(config_json) - # Validate req fields + # Validate req fields if ("network" not in config_json or "device_intf" not in config_json.get("network") or "internet_intf" not in config_json.get("network") or @@ -180,6 +179,10 @@ async def post_sys_config(self, request: Request, response: Response): False, "Configuration is missing required fields") + self._session.set_config(config_json) + + + # Catch JSON Decode error etc except JSONDecodeError: response.status_code = status.HTTP_400_BAD_REQUEST @@ -703,6 +706,18 @@ async def update_profile(self, request: Request, response: Response): profile_name = req_json.get("name") + # Error handling if profile name not in request + if not profile_name: + LOGGER.error("Missing 'name' in the request JSON") + response.status_code = status.HTTP_400_BAD_REQUEST + return self._generate_msg(False, "Invalid request received") + + # Error handling if 'questions' not in request + if "questions" not in req_json: + LOGGER.error("Missing 'questions' in the request JSON") + response.status_code = status.HTTP_400_BAD_REQUEST + return self._generate_msg(False, "Invalid request received") + # Check if profile exists profile = self.get_session().get_profile(profile_name) From 2a0babd43559f2f33ad966ff38e678614f010244 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Thu, 1 Aug 2024 14:59:22 +0100 Subject: [PATCH 09/30] fixed pylint --- testing/api/test_api.py | 52 ++++++++++------------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 88f0d0430..7699e6f1b 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -619,8 +619,9 @@ def test_system_shutdown_in_progress(testrun): # pylint: disable=W0613 # Tests for reports endpoints -def test_get_reports(testrun): # pylint: disable=W0613 - """Test for get reports endpoint""" +def test_get_reports_no_reports(testrun): # pylint: disable=W0613 + """Test get reports when no reports exist.""" + # Send a GET request to the /reports endpoint r = requests.get(f"{API}/reports", timeout=5) @@ -633,36 +634,13 @@ def test_get_reports(testrun): # pylint: disable=W0613 # Check if the response is a list assert isinstance(response, list) - # Iterate through each report - for report in response: - # Check if the report is dict - assert isinstance(report, dict) - # Check if "mac_adrr" key is in report - assert "mac_addr" in report - # Check if "device" key is in report - assert "device" in report - # Check if "status" key is in report - assert "status" in report - # Check if "started" key is in report - assert "started" in report - # Check if "finished" key is in report - assert "finished" in report - # Check if "tests" key is in report - assert "tests" in report - # Check if "report" key is in report - assert "report" in report - -def test_get_reports_no_reports(testrun): # pylint: disable=W0613 - """Test get reports when no reports exist.""" - r = requests.get(f"{API}/reports", timeout=5) - assert r.status_code == 200, f"Expected 200, got {r.status_code}" - response = r.json() - assert response == [], "Expected empty list when no reports exist" + # Check if the response is an empty list + assert response == [] @pytest.mark.skip() def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 """Initiate a test run, ensure report generation, and fetch the report.""" - # Step 1: Prepare and start the test run + payload = { "device": { "mac_addr": BASELINE_MAC_ADDR, @@ -685,12 +663,12 @@ def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 device_name = "test_device" start_test_device(device_name, BASELINE_MAC_ADDR) - # Step 3: Wait for the test to complete - max_retries = 300 # time limit + # Wait for the test to complete + max_retries = 300 for _ in range(max_retries): time.sleep(1) status = query_system_status().lower() - print(f"Current system status: {status}") # Additional logging + print(f"Current system status: {status}") if status in ["compliant", "non-compliant", "cancelled"]: break else: @@ -698,15 +676,15 @@ def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 stop_test_device(device_name) final_status = query_system_status().lower() # Get the final status print("Final system status:", final_status) # Log the final status - pytest.fail("Test run did not complete within the expected time. Final status: " + final_status) + pytest.fail("Test run did not complete. Final status: " + final_status) - # Step 4: Fetch the generated report + # Get request to retrieve the generated report r = requests.get(f"{API}/report/{BASELINE_MAC_ADDR}", timeout=5) assert r.status_code == 200, "Failed to fetch the report" report_data = r.json() print(f"Reports are {report_data}") - # Step 5: Stop the test device + # Stop the test device stop_test_device(device_name) def test_delete_report(testrun): # pylint: disable=W0613 @@ -1325,14 +1303,12 @@ def test_device_edit_device_with_mac_already_exists( assert r.status_code == 409 - def test_system_latest_version(testrun): # pylint: disable=W0613 r = requests.get(f"{API}/system/version", timeout=5) assert r.status_code == 200 updated_system_version = r.json()["update_available"] assert updated_system_version is False - def test_invalid_path_get(testrun): # pylint: disable=W0613 r = requests.get(f"{API}/blah/blah", timeout=5) response = r.json() @@ -1346,7 +1322,6 @@ def test_invalid_path_get(testrun): # pylint: disable=W0613 # validate structure assert set(dict_paths(mockito)) == set(dict_paths(response)) - def test_create_invalid_chars(empty_devices_dir, testrun): # pylint: disable=W0613 # local_delete_devices(ALL_DEVICES) # We must start test run with no devices in local/devices for this test @@ -1372,7 +1347,6 @@ def test_create_invalid_chars(empty_devices_dir, testrun): # pylint: disable=W06 print(r.text) print(r.status_code) - # Tests for profile endpoints def delete_all_profiles(): """Utility method to delete all profiles from risk_profiles folder""" @@ -1419,7 +1393,6 @@ def create_profile(file_name): # Return the profile return new_profile - @pytest.fixture() def reset_profiles(): """Delete the profiles before and after each test""" @@ -1686,7 +1659,6 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response - def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for delete profile""" From 18ef6569bacefbc15f1a944dc3efb9520438863c Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Fri, 2 Aug 2024 13:33:10 +0100 Subject: [PATCH 10/30] Added tests when update is available and 500 status code for '/system/version', test for system/modules --- testing/api/test_api.py | 167 ++++++++++++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 25 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 7699e6f1b..9407684fa 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -28,6 +28,8 @@ from typing import Iterator import pytest import requests +import responses +import datetime ALL_DEVICES = "*" API = "http://127.0.0.1:8000" @@ -617,6 +619,74 @@ def test_system_shutdown_in_progress(testrun): # pylint: disable=W0613 # Check if the response status code is 400 (test in progress) assert r.status_code == 400 +def test_system_latest_version(testrun): # pylint: disable=W0613 + """Test for testrun version when the latest version is installed""" + + # Send the get request to the API + r = requests.get(f"{API}/system/version", timeout=5) + + # Parse the response + response = r.json() + + # Check if status code is 200 (update available) + assert r.status_code == 200 + # Check if an update is available + assert response["update_available"] is False + +@responses.activate +def test_system_update_available(testrun): # pylint: disable=W0613 + """Test for testrun version when update is available""" + + # Mock the API response when the update is available + responses.add( + responses.GET, + f"{API}/system/version", + json={ + "installed_version": "3.1", + "update_available": True, + "latest_version": "3.1.1", + "latest_version_url": + ("https://github.com/google/testrun/releases/tag/v3.1.1") + }, + status=200 + ) + + # Send the get request to the API + r = requests.get(f"{API}/system/version", timeout=5) + + # Parse the response + response = r.json() + + # Check if status code is 200 (update available) + assert r.status_code == 200 + # Check if an update is available + assert response["update_available"] is True + +@responses.activate +def test_system_version_cannot_be_obtained(testrun): # pylint: disable=W0613 + """Test when the current version cannot be obtained""" + + # Mock the API response when the current version cannot be obtained + responses.add( + responses.GET, + f"{API}/system/version", + json={"error": "Could not fetch current version"}, + status=500 + ) + + # Send the get request to the API + r = requests.get(f"{API}/system/version", timeout=5) + + # Parse the JSON response + response = r.json() + + + # Check if the response status code is 500 + assert r.status_code == 500 + + # Check if the error in response + assert "error" in response + # Tests for reports endpoints def test_get_reports_no_reports(testrun): # pylint: disable=W0613 @@ -638,8 +708,8 @@ def test_get_reports_no_reports(testrun): # pylint: disable=W0613 assert response == [] @pytest.mark.skip() -def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 - """Initiate a test run, ensure report generation, and fetch the report.""" +def get_report_one_report(testrun): # pylint: disable=W0613 + """Initiate a test run, ensure report generation, and get the report.""" payload = { "device": { @@ -647,10 +717,11 @@ def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 "firmware": "asd", "test_modules": { "dns": {"enabled": False}, - "connection": {"enabled": True}, - "ntp": {"enabled": False}, - "baseline": {"enabled": False}, - "nmap": {"enabled": False} + "connection": {"enabled": False}, + "ntp": {"enabled": True}, + "services": {"enabled": False}, + "protocol": {"enabled": False}, + "tls": {"enabled": False} } } } @@ -674,30 +745,53 @@ def run_test_and_get_report(testing_devices, testrun): # pylint: disable=W0613 else: # Timeout reached without a valid end state stop_test_device(device_name) - final_status = query_system_status().lower() # Get the final status - print("Final system status:", final_status) # Log the final status + # Get the final status + final_status = query_system_status().lower() + print("Final system status:", final_status) + # Log the final status pytest.fail("Test run did not complete. Final status: " + final_status) + # Get the current timestamp in the expected format + timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + # Get request to retrieve the generated report - r = requests.get(f"{API}/report/{BASELINE_MAC_ADDR}", timeout=5) - assert r.status_code == 200, "Failed to fetch the report" + r = requests.get(f"{API}/report/{device_name}/{timestamp}", timeout=5) + # Parse the json report_data = r.json() + print(f"Reports are {report_data}") + assert r.status_code == 200 # Stop the test device stop_test_device(device_name) -def test_delete_report(testrun): # pylint: disable=W0613 - """Test the delete report endpoint""" - pass +# def test_get_report_no_report(testrun): # pylint: disable=W0613 + device_name = "test" + timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") -def test_get_report(testrun): # pylint: disable=W0613 - """Test the get report endpoint""" - pass + # Get request to retrieve the generated report + r = requests.get(f"{API}/report/{device_name}/{timestamp}", timeout=5) + # Parse the json + response = r.json() + print(f"response is {response}") -def test_export_report(testrun): # pylint: disable=W0613 - """Test the export report endpoint""" - pass + # Check if the response is a list + assert "Not Found" in response.values() + + # Check if status code is 404 (Report Not Found) + assert r.status_code == 404 + +# def test_delete_report(testrun): # pylint: disable=W0613 +# """Test the delete report endpoint""" +# pass + +# def test_get_report(testrun): # pylint: disable=W0613 +# """Test the get report endpoint""" +# pass + +# def test_export_report(testrun): # pylint: disable=W0613 + # """Test the export report endpoint""" + # pass # Tests for device endpoints @@ -1303,12 +1397,6 @@ def test_device_edit_device_with_mac_already_exists( assert r.status_code == 409 -def test_system_latest_version(testrun): # pylint: disable=W0613 - r = requests.get(f"{API}/system/version", timeout=5) - assert r.status_code == 200 - updated_system_version = r.json()["update_available"] - assert updated_system_version is False - def test_invalid_path_get(testrun): # pylint: disable=W0613 r = requests.get(f"{API}/blah/blah", timeout=5) response = r.json() @@ -1347,6 +1435,35 @@ def test_create_invalid_chars(empty_devices_dir, testrun): # pylint: disable=W06 print(r.text) print(r.status_code) +def test_get_test_modules(testrun): # pylint: disable=W0613 + """Test the /system/modules endpoint to check the test modules""" + + # Send a GET request to the API endpoint + r = requests.get(f"{API}/system/modules", timeout=5) + + # Check if status code is 200 (OK) + assert r.status_code == 200 + + # Parse the JSON response + response = r.json() + + # Check if the response is a list + assert isinstance(response, list) + + # Define the expected modules + expected_modules = [ + "Connection", + "Services", + "NTP", + "DNS", + "Protocol", + "TLS" + ] + + # Check if all expected modules are in the response + for module in expected_modules: + assert module in response + # Tests for profile endpoints def delete_all_profiles(): """Utility method to delete all profiles from risk_profiles folder""" From 9f436ad137d7aba424d5ca2f0ff7aeecf28d053a Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Fri, 2 Aug 2024 13:36:41 +0100 Subject: [PATCH 11/30] added responses library in requirements.txt --- framework/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/framework/requirements.txt b/framework/requirements.txt index 402009ef9..0484905ee 100644 --- a/framework/requirements.txt +++ b/framework/requirements.txt @@ -21,6 +21,8 @@ pydantic==2.7.1 # Requirements for testing pytest==7.4.4 pytest-timeout==2.2.0 +responses==0.25.3 + # Requirements for the report markdown==3.5.2 From 87904c96068e93a69d615cf0797473282b100881 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Fri, 2 Aug 2024 13:45:40 +0100 Subject: [PATCH 12/30] fixed the requested changes in api.py --- framework/python/src/api/api.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index f6c44b2fe..6e21c2813 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -169,7 +169,7 @@ async def post_sys_config(self, request: Request, response: Response): config = (await request.body()).decode("UTF-8") config_json = json.loads(config) - # Validate req fields + # Validate req fields if ("network" not in config_json or "device_intf" not in config_json.get("network") or "internet_intf" not in config_json.get("network") or @@ -180,9 +180,7 @@ async def post_sys_config(self, request: Request, response: Response): "Configuration is missing required fields") self._session.set_config(config_json) - - - + # Catch JSON Decode error etc except JSONDecodeError: response.status_code = status.HTTP_400_BAD_REQUEST From c44569179d535299b9cd38f25e36e0c1707c128e Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Fri, 2 Aug 2024 15:13:34 +0100 Subject: [PATCH 13/30] Renamed the load_profile method to load_json and changed the logic to allow to load any json based on file name and relative path, corrected the new lines issues --- testing/api/test_api.py | 122 ++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 41 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 9407684fa..81066fd33 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -100,6 +100,17 @@ def docker_logs(device_name): ) print(cmd.stdout) +def load_json(file_name, directory): + """Utility method to load json files' """ + # Construct the base path relative to the main folder + base_path = Path(__file__).resolve().parent.parent.parent + # Construct the full file path + file_path = base_path / directory / file_name + + # Open the file in read mode + with open(file_path, "r", encoding="utf-8") as file: + # Return the file content + return json.load(file) @pytest.fixture def empty_devices_dir(): @@ -232,7 +243,7 @@ def local_get_devices(): @pytest.fixture() def restore_config(): """Restore the original configuration (system.json) after the test""" - yield + # Restore system.json from 'testing/api/' after the test if os.path.exists(SYSTEM_CONFIG_RESTORE_PATH): shutil.copy(SYSTEM_CONFIG_RESTORE_PATH, SYSTEM_CONFIG_PATH) @@ -242,13 +253,16 @@ def test_get_system_interfaces(testrun): # pylint: disable=W0613 # Send a GET request to the API to retrieve system interfaces r = requests.get(f"{API}/system/interfaces", timeout=5) + + # Check if status code is 200 (OK) + assert r.status_code == 200 + # Parse the JSON response response = r.json() + # Retrieve the actual network interfaces local_interfaces = get_network_interfaces() - # Check if status code is 200 (OK) - assert r.status_code == 200, f"status code is {r.status_code}" # Check if the key are in the response assert set(response.keys()) == set(local_interfaces) # Ensure that all values in the response are strings @@ -298,7 +312,7 @@ def test_update_system_config(testrun, restore_config): # pylint: disable=W0613 def test_update_system_config_invalid_config(testrun, restore_config): # pylint: disable=W0613 """Test invalid configuration file for update system configuration""" - # Configuration data to update + # Configuration data to update with missing "log_level" field updated_system_config = { "network": { "device_intf": "updated_endev0a", @@ -316,15 +330,12 @@ def test_update_system_config_invalid_config(testrun, restore_config): # pylint: def test_get_system_config(testrun): # pylint: disable=W0613 """Tests get system configuration endpoint ('/system/config')""" + # Send a GET request to the API to retrieve system configuration r = requests.get(f"{API}/system/config", timeout=5) - # Open the local system configuration file - with open( - SYSTEM_CONFIG_PATH, - encoding="utf-8" - ) as f: - local_config = json.load(f) + # Load system configuration file + local_config = load_json("system.json", directory="local") # Parse the JSON response api_config = r.json() @@ -365,6 +376,7 @@ def test_start_testrun_started_successfully( "baseline": {"enabled": False}, "nmap": {"enabled": False} }}} + # Send the post request r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) @@ -606,6 +618,7 @@ def test_system_shutdown_in_progress(testrun): # pylint: disable=W0613 } } } + # Start a test r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) @@ -1436,7 +1449,7 @@ def test_create_invalid_chars(empty_devices_dir, testrun): # pylint: disable=W06 print(r.status_code) def test_get_test_modules(testrun): # pylint: disable=W0613 - """Test the /system/modules endpoint to check the test modules""" + """Test the /system/modules endpoint to get the test modules""" # Send a GET request to the API endpoint r = requests.get(f"{API}/system/modules", timeout=5) @@ -1467,6 +1480,8 @@ def test_get_test_modules(testrun): # pylint: disable=W0613 # Tests for profile endpoints def delete_all_profiles(): """Utility method to delete all profiles from risk_profiles folder""" + + # Assign the profiles directory profiles_path = Path(PROFILES_DIRECTORY) try: @@ -1491,8 +1506,10 @@ def delete_all_profiles(): def create_profile(file_name): """Utility method to create the profile""" + # Load the profile - new_profile = load_profile(file_name) + new_profile = load_json(file_name, directory="testing/api/profiles") + # Assign the profile name to profile_name profile_name = new_profile["name"] @@ -1513,9 +1530,12 @@ def create_profile(file_name): @pytest.fixture() def reset_profiles(): """Delete the profiles before and after each test""" + # Delete before the test delete_all_profiles() + yield + # Delete after the test delete_all_profiles() @@ -1525,15 +1545,6 @@ def add_profile(): # Returning the reference to create_profile return create_profile -def load_profile(file_name): - """Utility method to load the profiles from 'testing/api/profiles' """ - # Construct the file path - file_path = os.path.join(os.path.dirname(__file__), "profiles", file_name) - # Open the file in read mode - with open(file_path, "r", encoding="utf-8") as file: - # Return the file content - return json.load(file) - def profile_exists(profile_name): """Utility method to check if profile exists""" # Send the get request @@ -1629,10 +1640,11 @@ def test_get_profiles(testrun, reset_profiles, add_profile): # pylint: disable= # Send the get request to "/profiles" endpoint r = requests.get(f"{API}/profiles", timeout=5) - # Check if status code is 200 (OK) - assert r.status_code == 200 # Parse the response (profiles) response = r.json() + + # Check if status code is 200 (OK) + assert r.status_code == 200 # Check if response is a list assert isinstance(response, list) # Check if response contains two profiles @@ -1642,7 +1654,8 @@ def test_create_profile(testrun, reset_profiles): # pylint: disable=W0613 """Test for create profile if not exists""" # Load the profile - new_profile = load_profile("new_profile.json") + new_profile = load_json("new_profile.json", directory="testing/api/profiles") + # Assign the profile name to profile_name profile_name = new_profile["name"] @@ -1655,8 +1668,10 @@ def test_create_profile(testrun, reset_profiles): # pylint: disable=W0613 # Check if status code is 201 (Created) assert r.status_code == 201 + # Parse the response response = r.json() + # Check if "success" key in response assert "success" in response @@ -1665,6 +1680,7 @@ def test_create_profile(testrun, reset_profiles): # pylint: disable=W0613 # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the response profiles = r.json() @@ -1672,19 +1688,23 @@ def test_create_profile(testrun, reset_profiles): # pylint: disable=W0613 created_profile = next( (p for p in profiles if p["name"] == profile_name), None ) + # Check if profile was created assert created_profile is not None def test_update_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 - """test for update profile when exists""" + """Test for update profile when exists""" # Load the new profile using add_profile fixture new_profile = add_profile("new_profile.json") - # Load the updated profile using load_profile utility method - updated_profile = load_profile("updated_profile.json") + + # Load the updated profile using load_json utility method + updated_profile = load_json("updated_profile.json", + directory="testing/api/profiles") # Assign the new_profile name profile_name = new_profile["name"] + # Assign the updated_profile name updated_profile_name = updated_profile["rename"] @@ -1700,8 +1720,10 @@ def test_update_profile(testrun, reset_profiles, add_profile): # pylint: disable # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the response response = r.json() + # Check if "success" key in response assert "success" in response @@ -1710,6 +1732,7 @@ def test_update_profile(testrun, reset_profiles, add_profile): # pylint: disable # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the response profiles = r.json() @@ -1723,10 +1746,13 @@ def test_update_profile(testrun, reset_profiles, add_profile): # pylint: disable def test_update_profile_invalid_json(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for update profile invalid JSON payload (no 'name')""" + # Load the new profile using add_profile fixture add_profile("new_profile.json") - # Load the updated profile using load_profile utility method - updated_profile = load_profile("profile_invalid_format.json") + + # Load the updated profile using load_json utility method + updated_profile = load_json("profile_invalid_format.json", + directory="testing/api/profiles") # Send the post request to update the profile r = requests.post( @@ -1734,18 +1760,21 @@ def test_update_profile_invalid_json(testrun, reset_profiles, add_profile): # py data=json.dumps(updated_profile), timeout=5) - # Check if status code is 400 (Bad request) - assert r.status_code == 400 # Parse the response response = r.json() + + # Check if status code is 400 (Bad request) + assert r.status_code == 400 + # Check if "error" key in response assert "error" in response def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable=W0613 """Test for create profile invalid JSON payload """ - # Load the profile using load_profile utility method - new_profile = load_profile("profile_invalid_format.json") + # Load the profile using load_json utility method + new_profile = load_json("profile_invalid_format.json", + directory="testing/api/profiles") # Send the post request to update the profile r = requests.post( @@ -1753,10 +1782,12 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable data=json.dumps(new_profile), timeout=5) - # Check if status code is 400 (Bad request) - assert r.status_code == 400 # Parse the response response = r.json() + + # Check if status code is 400 (Bad request) + assert r.status_code == 400 + # Check if "error" key in response assert "error" in response @@ -1769,10 +1800,12 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable data=json.dumps(new_profile_2), timeout=5) - # Check if status code is 400 (Bad request) - assert r.status_code == 400 # Parse the response response = r.json() + + # Check if status code is 400 (Bad request) + assert r.status_code == 400 + # Check if "error" key in response assert "error" in response @@ -1781,6 +1814,7 @@ def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable # Assign the profile from the fixture profile_to_delete = add_profile("new_profile.json") + # Assign the profile name profile_name = profile_to_delete["name"] @@ -1792,8 +1826,10 @@ def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the JSON response response = r.json() + # Check if the response contains "success" key assert "success" in response @@ -1802,6 +1838,7 @@ def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the JSON response profiles = r.json() @@ -1815,8 +1852,9 @@ def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable def test_delete_profile_no_profile(testrun, reset_profiles): # pylint: disable=W0613 """Test delete profile if the profile does not exists""" - # Load the profile to delete - profile_to_delete = load_profile("new_profile.json") + + # Assign the profile to delete + profile_to_delete = {"name": "New Profile"} # Delete the profile r = requests.delete( @@ -1830,7 +1868,7 @@ def test_delete_profile_no_profile(testrun, reset_profiles): # pylint: disable=W def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable=W0613 """Test for delete profile wrong JSON payload""" - profile_to_delete = {} #load_profile("profile_invalid_format.json") + profile_to_delete = {} # Delete the profile r = requests.delete( @@ -1843,10 +1881,12 @@ def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if status code is 400 (bad request) assert r.status_code == 400 + # Check if "error" key in response assert "error" in response - profile_to_delete_2 = load_profile("profile_invalid_format.json") + profile_to_delete_2 = load_json("profile_invalid_format.json", + directory="testing/api/profiles") # Delete the profile r = requests.delete( From d7cd17576dc833497a3cbad1895a51c236c2e454 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Fri, 2 Aug 2024 15:29:11 +0100 Subject: [PATCH 14/30] updated restore_config fixture to run after the test --- testing/api/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 81066fd33..d5cb759fe 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -243,6 +243,7 @@ def local_get_devices(): @pytest.fixture() def restore_config(): """Restore the original configuration (system.json) after the test""" + yield # Restore system.json from 'testing/api/' after the test if os.path.exists(SYSTEM_CONFIG_RESTORE_PATH): From eadfcbdd7ca261af65db9edecb5454b4d47e8a70 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Fri, 2 Aug 2024 16:47:52 +0100 Subject: [PATCH 15/30] added test for create/update profile (500 error) --- testing/api/test_api.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index d5cb759fe..716064cd2 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -1810,6 +1810,32 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response +@responses.activate +def test_update_profile_internal_server_error(testrun): # pylint: disable=W0613 + """Test for create/update profile causing internal server error.""" + + # Mock the POST request for create/update profiles API response + responses.add( + responses.POST, + f"{API}/profiles", + json={"error": "An error occurred whilst creating or updating a profile"}, + status=500 + ) + + # Send the POST request to create/update the profile + r = requests.post(f"{API}/profiles", + json={"name": "New Profile", "questions": []}, + timeout=5) + + # Parse the json response + response = r.json() + + # Check if status code is 500 (Internal Server Error) + assert r.status_code == 500 + + # Check if "error" key in response + assert "error" in response + def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for delete profile""" @@ -1902,3 +1928,5 @@ def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable assert r.status_code == 400 # Check if "error" key in response assert "error" in response + + From 5fe65334627795526e11a7a41a624b137fff8dae Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Mon, 5 Aug 2024 09:18:08 +0100 Subject: [PATCH 16/30] fixed pylint --- framework/python/src/api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 6e21c2813..5547420ac 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -180,7 +180,7 @@ async def post_sys_config(self, request: Request, response: Response): "Configuration is missing required fields") self._session.set_config(config_json) - + # Catch JSON Decode error etc except JSONDecodeError: response.status_code = status.HTTP_400_BAD_REQUEST From 9212935e0a33b939dcb76dcd669f0414ad388382 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Mon, 5 Aug 2024 09:23:27 +0100 Subject: [PATCH 17/30] fixed spacing, removed get_report_one_report --- testing/api/test_api.py | 95 ----------------------------------------- 1 file changed, 95 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 716064cd2..1ae742ab1 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -29,7 +29,6 @@ import pytest import requests import responses -import datetime ALL_DEVICES = "*" API = "http://127.0.0.1:8000" @@ -49,21 +48,18 @@ def pretty_print(dictionary: dict): """ Pretty print dictionary """ print(json.dumps(dictionary, indent=4)) - def query_system_status() -> str: """Query system status from API and returns this""" r = requests.get(f"{API}/system/status", timeout=5) response = r.json() return response["status"] - def query_test_count() -> int: """Queries status and returns number of test results""" r = requests.get(f"{API}/system/status", timeout=5) response = r.json() return len(response["tests"]["results"]) - def start_test_device( device_name, mac_address, image_name="test-run/ci_device_1", args="" ): @@ -78,7 +74,6 @@ def start_test_device( ) print(cmd.stdout) - def stop_test_device(device_name): """ Stop docker container with given name """ cmd = subprocess.run( @@ -117,7 +112,6 @@ def empty_devices_dir(): """ Use e,pty devices directory """ local_delete_devices(ALL_DEVICES) - @pytest.fixture def testing_devices(): """ Use devices from the testing/device_configs directory """ @@ -129,7 +123,6 @@ def testing_devices(): ) return local_get_devices() - @pytest.fixture def testrun(request): # pylint: disable=W0613 """ Start intstance of testrun """ @@ -180,7 +173,6 @@ def testrun(request): # pylint: disable=W0613 ) print(cmd.stdout) - def until_true(func: Callable, message: str, timeout: int): """ Blocks until given func returns True @@ -194,7 +186,6 @@ def until_true(func: Callable, message: str, timeout: int): time.sleep(1) raise TimeoutError(f"Timed out waiting {timeout}s for {message}") - def dict_paths(thing: dict, stem: str = "") -> Iterator[str]: """Returns json paths (in dot notation) from a given dictionary""" for k, v in thing.items(): @@ -204,7 +195,6 @@ def dict_paths(thing: dict, stem: str = "") -> Iterator[str]: else: yield path - def get_network_interfaces(): """return list of network interfaces on machine @@ -219,7 +209,6 @@ def get_network_interfaces(): ifaces.append(i.stem) return ifaces - def local_delete_devices(path): """ Deletes all local devices """ @@ -229,7 +218,6 @@ def local_delete_devices(path): else: shutil.rmtree(thing) - def local_get_devices(): """ Returns path to device configs of devices in local/devices directory""" return sorted( @@ -721,80 +709,6 @@ def test_get_reports_no_reports(testrun): # pylint: disable=W0613 # Check if the response is an empty list assert response == [] -@pytest.mark.skip() -def get_report_one_report(testrun): # pylint: disable=W0613 - """Initiate a test run, ensure report generation, and get the report.""" - - payload = { - "device": { - "mac_addr": BASELINE_MAC_ADDR, - "firmware": "asd", - "test_modules": { - "dns": {"enabled": False}, - "connection": {"enabled": False}, - "ntp": {"enabled": True}, - "services": {"enabled": False}, - "protocol": {"enabled": False}, - "tls": {"enabled": False} - } - } - } - - # Send the post request to start the test - r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) - assert r.status_code == 200 - - # Step 2: Start the device for the test - device_name = "test_device" - start_test_device(device_name, BASELINE_MAC_ADDR) - - # Wait for the test to complete - max_retries = 300 - for _ in range(max_retries): - time.sleep(1) - status = query_system_status().lower() - print(f"Current system status: {status}") - if status in ["compliant", "non-compliant", "cancelled"]: - break - else: - # Timeout reached without a valid end state - stop_test_device(device_name) - # Get the final status - final_status = query_system_status().lower() - print("Final system status:", final_status) - # Log the final status - pytest.fail("Test run did not complete. Final status: " + final_status) - - # Get the current timestamp in the expected format - timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - - # Get request to retrieve the generated report - r = requests.get(f"{API}/report/{device_name}/{timestamp}", timeout=5) - # Parse the json - report_data = r.json() - - print(f"Reports are {report_data}") - assert r.status_code == 200 - - # Stop the test device - stop_test_device(device_name) - -# def test_get_report_no_report(testrun): # pylint: disable=W0613 - device_name = "test" - timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - - # Get request to retrieve the generated report - r = requests.get(f"{API}/report/{device_name}/{timestamp}", timeout=5) - # Parse the json - response = r.json() - print(f"response is {response}") - - # Check if the response is a list - assert "Not Found" in response.values() - - # Check if status code is 404 (Report Not Found) - assert r.status_code == 404 - # def test_delete_report(testrun): # pylint: disable=W0613 # """Test the delete report endpoint""" # pass @@ -1118,7 +1032,6 @@ def test_start_system_not_configured_correctly( timeout=10) assert r.status_code == 500 - def test_start_device_not_found(empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 device_1 = { @@ -1151,7 +1064,6 @@ def test_start_device_not_found(empty_devices_dir, # pylint: disable=W0613 timeout=10) assert r.status_code == 404 - def test_start_missing_device_information( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1180,7 +1092,6 @@ def test_start_missing_device_information( timeout=10) assert r.status_code == 400 - def test_create_device_already_exists( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1210,7 +1121,6 @@ def test_create_device_already_exists( print(r.text) assert r.status_code == 409 - def test_create_device_invalid_json( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1223,7 +1133,6 @@ def test_create_device_invalid_json( print(r.text) assert r.status_code == 400 - def test_create_device_invalid_request( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1234,7 +1143,6 @@ def test_create_device_invalid_request( print(r.text) assert r.status_code == 400 - def test_device_edit_device( testing_devices, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1283,7 +1191,6 @@ def test_device_edit_device( assert updated_device_api["model"] == new_model assert updated_device_api["test_modules"] == new_test_modules - def test_device_edit_device_not_found( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1321,7 +1228,6 @@ def test_device_edit_device_not_found( assert r.status_code == 404 - def test_device_edit_device_incorrect_json_format( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 @@ -1354,7 +1260,6 @@ def test_device_edit_device_incorrect_json_format( assert r.status_code == 400 - def test_device_edit_device_with_mac_already_exists( empty_devices_dir, # pylint: disable=W0613 testrun): # pylint: disable=W0613 From 22dfbe047bfdafca43ac75374b3c53fc6db40cf5 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Mon, 5 Aug 2024 14:49:35 +0100 Subject: [PATCH 18/30] added tests: 500 error for delete '/profiles', 500 error for 'profles/format', 400, 404, 409 for '/system/start' --- testing/api/test_api.py | 162 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 153 insertions(+), 9 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 1ae742ab1..9fe60c04f 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -330,7 +330,7 @@ def test_get_system_config(testrun): # pylint: disable=W0613 api_config = r.json() # Check if status code is 200 (OK) - assert r.status_code == 200, f"status code is {r.status_code}" + assert r.status_code == 200 # Validate structure assert set(dict_paths(api_config)) | set(dict_paths(local_config)) == set( @@ -349,10 +349,8 @@ def test_get_system_config(testrun): # pylint: disable=W0613 == api_config["network"]["internet_intf"] ) -def test_start_testrun_started_successfully( - testing_devices, # pylint: disable=W0613 - testrun): # pylint: disable=W0613 - """Test if testrun started successfully """ +def test_start_testrun_started_successfully(testing_devices, testrun): # pylint: disable=W0613 + """Test for testrun started successfully """ # Payload with device details payload = {"device": { @@ -372,6 +370,96 @@ def test_start_testrun_started_successfully( # Check if the response status code is 200 (OK) assert r.status_code == 200 + # Parse the json response + response = r.json() + + # Check that device is in response + assert "device" in response + + # Check that mac_addr in response + assert "mac_addr" in response["device"] + + # Check that firmware in response + assert "firmware" in response["device"] + +def test_start_testrun_missing_device(testing_devices, testrun): # pylint: disable=W0613 + """Test for missing device when testrun is started """ + + # Payload empty dict (no device) + payload = {} + + # Send the post request + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + # Check if the response status code is 400 (bad request) + assert r.status_code == 400 + + # Parse the json response + response = r.json() + + # Check if 'error' in response + assert "error" in response + +def test_start_testrun_already_started(testing_devices, testrun): # pylint: disable=W0613 + """Test for testrun already started """ + + # Payload with device details + payload = {"device": { + "mac_addr": BASELINE_MAC_ADDR, + "firmware": "asd", + "test_modules": { + "dns": {"enabled": False}, + "connection": {"enabled": True}, + "ntp": {"enabled": False}, + "baseline": {"enabled": False}, + "nmap": {"enabled": False} + }}} + + # Send the post request (start test) + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + # Check if the response status code is 200 (OK) + assert r.status_code == 200 + + # Send the second post request (start test again) + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + # Parse the json response + response = r.json() + + # Check if the response status code is 409 (Conflict) + assert r.status_code == 409 + + # Check if 'error' in response + assert "error" in response + +def test_start_testrun_device_not_found(testing_devices, testrun): # pylint: disable=W0613 + """Test for start testrun device not found """ + + # Payload with device details with no mac address assigned + payload = {"device": { + "mac_addr": "", + "firmware": "asd", + "test_modules": { + "dns": {"enabled": False}, + "connection": {"enabled": True}, + "ntp": {"enabled": False}, + "baseline": {"enabled": False}, + "nmap": {"enabled": False} + }}} + + # Send the post request + r = requests.post(f"{API}/system/start", data=json.dumps(payload), timeout=10) + + # Check if the response status code is 404 (not found) + assert r.status_code == 404 + + # Parse the json response + response = r.json() + + # Check if 'error' in response + assert "error" in response + # Currently not working due to blocking during monitoring period @pytest.mark.skip() def test_status_in_progress(testing_devices, testrun): # pylint: disable=W0613 @@ -1465,12 +1553,16 @@ def profile_exists(profile_name): def test_get_profiles_format(testrun): # pylint: disable=W0613 """Test profiles format""" + # Send the get request r = requests.get(f"{API}/profiles/format", timeout=5) + # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the response response = r.json() + # Check if the response is a list assert isinstance(response, list) @@ -1479,6 +1571,30 @@ def test_get_profiles_format(testrun): # pylint: disable=W0613 assert "question" in item assert "type" in item +@responses.activate +def test_get_profiles_format_internal_server_error(testrun): # pylint: disable=W0613 + """Test for get_profiles_format causing internal server error""" + + # Mock the response for GET request for getting profiles format + responses.add( + responses.GET, + f"{API}/profiles/format", + json={"error": "Testrun could not load the risk assessment format"}, + status=500 + ) + + # Send the get request + r = requests.get(f"{API}/profiles/format", timeout=5) + + # Check if status code is 500 (Internal Server Error) + assert r.status_code == 500 + + # Parse the response + response = r.json() + + # Check if "error" key in response + assert "error" in response + def test_get_profiles(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for get profiles (no profile, one profile, two profiles)""" @@ -1716,7 +1832,7 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable assert "error" in response @responses.activate -def test_update_profile_internal_server_error(testrun): # pylint: disable=W0613 +def test_create_update_profile_internal_server_error(testrun): # pylint: disable=W0613 """Test for create/update profile causing internal server error.""" # Mock the POST request for create/update profiles API response @@ -1732,12 +1848,12 @@ def test_update_profile_internal_server_error(testrun): # pylint: disable=W0613 json={"name": "New Profile", "questions": []}, timeout=5) - # Parse the json response - response = r.json() - # Check if status code is 500 (Internal Server Error) assert r.status_code == 500 + # Parse the json response + response = r.json() + # Check if "error" key in response assert "error" in response @@ -1834,4 +1950,32 @@ def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response +@responses.activate +def test_delete_profile_internal_server_error(testrun): # pylint: disable=W0613 + """Test for create/update profile causing internal server error""" + + # Assign the profile to delete + profile_to_delete = {"name": "New Profile"} + + # Mock the response for DELETE request for deleting a profile + responses.add( + responses.DELETE, + f"{API}/profiles", + json={"error": "An error occurred whilst deleting the profile"}, + status=500 + ) + + # Send the DELETE request to delete the profile + r = requests.delete(f"{API}/profiles", + json=profile_to_delete, + timeout=5) + + # Check if status code is 500 (Internal Server Error) + assert r.status_code == 500 + + # Parse the JSON response + response = r.json() + + # Check if "error" key in response + assert "error" in response From 7b49ece20e8a91b72aea7a9dfc7e12976fcedec0 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Tue, 6 Aug 2024 16:48:22 +0100 Subject: [PATCH 19/30] modified the tests for 500 response --- testing/api/test_api.py | 100 ++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 60 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 9fe60c04f..298389f76 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -30,6 +30,19 @@ import requests import responses +#import sys + +# from unittest.mock import patch, MagicMock + +# # Get the directory of the current script +# current_dir = os.path.dirname(os.path.abspath(__file__)) + +# # Define the relative path to the desired directory +# relative_path = os.path.join(current_dir, '../../framework/python/src') + +# # Append the relative path to sys.path +# sys.path.append(relative_path) + ALL_DEVICES = "*" API = "http://127.0.0.1:8000" LOG_PATH = "/tmp/testrun.log" @@ -1571,30 +1584,6 @@ def test_get_profiles_format(testrun): # pylint: disable=W0613 assert "question" in item assert "type" in item -@responses.activate -def test_get_profiles_format_internal_server_error(testrun): # pylint: disable=W0613 - """Test for get_profiles_format causing internal server error""" - - # Mock the response for GET request for getting profiles format - responses.add( - responses.GET, - f"{API}/profiles/format", - json={"error": "Testrun could not load the risk assessment format"}, - status=500 - ) - - # Send the get request - r = requests.get(f"{API}/profiles/format", timeout=5) - - # Check if status code is 500 (Internal Server Error) - assert r.status_code == 500 - - # Parse the response - response = r.json() - - # Check if "error" key in response - assert "error" in response - def test_get_profiles(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for get profiles (no profile, one profile, two profiles)""" @@ -1831,31 +1820,20 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response -@responses.activate -def test_create_update_profile_internal_server_error(testrun): # pylint: disable=W0613 - """Test for create/update profile causing internal server error.""" +def test_create_update_profile_missing_answer(testrun): # pylint: disable=W0613 + """Test for create/update profile causing bad request.""" - # Mock the POST request for create/update profiles API response - responses.add( - responses.POST, - f"{API}/profiles", - json={"error": "An error occurred whilst creating or updating a profile"}, - status=500 - ) + # Load the json file + profile = load_json("no_answer.json", directory="testing/api/profiles") - # Send the POST request to create/update the profile - r = requests.post(f"{API}/profiles", - json={"name": "New Profile", "questions": []}, - timeout=5) - - # Check if status code is 500 (Internal Server Error) - assert r.status_code == 500 + # Send teh post request + r = requests.post(f"{API}/profiles", data=json.dumps(profile), timeout=5) # Parse the json response - response = r.json() + print(r.json()) - # Check if "error" key in response - assert "error" in response + # Check if status code is 400 (bad request) + assert r.status_code == 400 def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for delete profile""" @@ -1950,32 +1928,34 @@ def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response -@responses.activate -def test_delete_profile_internal_server_error(testrun): # pylint: disable=W0613 - """Test for create/update profile causing internal server error""" +def test_delete_profile_internal_server_error(testrun, # pylint: disable=W0613 + reset_profiles, # pylint: disable=W0613 + add_profile ): + """Test for delete profile causing internal server error""" - # Assign the profile to delete - profile_to_delete = {"name": "New Profile"} + # Assign the profile from the fixture + profile_to_delete = add_profile("new_profile.json") - # Mock the response for DELETE request for deleting a profile - responses.add( - responses.DELETE, - f"{API}/profiles", - json={"error": "An error occurred whilst deleting the profile"}, - status=500 - ) + # Assign the profile name to profile_name + profile_name = profile_to_delete["name"] + + # Construct the path to the profile JSON file in local/risk_profiles + risk_profile_path = os.path.join(PROFILES_DIRECTORY, f"{profile_name}.json") + + # Delete the profile JSON file before making the DELETE request + if os.path.exists(risk_profile_path): + os.remove(risk_profile_path) # Send the DELETE request to delete the profile r = requests.delete(f"{API}/profiles", - json=profile_to_delete, + json={"name": profile_to_delete["name"]}, timeout=5) # Check if status code is 500 (Internal Server Error) assert r.status_code == 500 - # Parse the JSON response + # Parse the json response response = r.json() - # Check if "error" key in response + # Check if error in response assert "error" in response - From 5f886f64e0a3ed2f0863bca0bc000a715185f2d8 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Tue, 6 Aug 2024 16:51:24 +0100 Subject: [PATCH 20/30] added new profile with missing 'answer' --- testing/api/profiles/no_answer.json | 143 ++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 testing/api/profiles/no_answer.json diff --git a/testing/api/profiles/no_answer.json b/testing/api/profiles/no_answer.json new file mode 100644 index 000000000..9f3c033d3 --- /dev/null +++ b/testing/api/profiles/no_answer.json @@ -0,0 +1,143 @@ +{ + "name": "New Profile", + "status": "Valid", + "created": "2024-05-23 12:38:26", + "version": "v1.3", + "questions": [ + { + "question": "What type of device is this?", + "type": "select", + "options": [ + "IoT Sensor", + "IoT Controller", + "Smart Device", + "Something else" + ], + "validation": { + "required": true + } + }, + { + "question": "How will this device be used at Google?", + "type": "text-long", + "validation": { + "max": "128", + "required": true + } + }, + { + "question": "What is the email of the device owner(s)?", + "type": "email-multiple", + "answer": "boddey@google.com, cmeredith@google.com", + "validation": { + "required": true, + "max": "128" + } + }, + { + "question": "Is this device going to be managed by Google or a third party?", + "type": "select", + "options": [ + "Google", + "Third Party" + ], + "answer": "Google", + "validation": { + "required": true + } + }, + { + "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", + "type": "select", + "options": [ + "Yes", + "No", + "N/A" + ], + "default": "N/A", + "answer": "Yes", + "validation": { + "required": true + } + }, + { + "question": "Are any of the following statements true about your device?", + "description": "This tells us about the data your device will collect", + "type": "select-multiple", + "answer": [ + 0, + 2 + ], + "options": [ + "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", + "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", + "The device stream confidential business data in real-time (seconds)?" + ] + }, + { + "question": "Which of the following statements are true about this device?", + "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", + "type": "select-multiple", + "answer": [ + 0, + 1, + 5 + ], + "options": [ + "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", + "Data transmission occurs across less-trusted networks (e.g. the internet).", + "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", + "A confidentiality breach during transmission would have a substantial negative impact", + "The device encrypts data during transmission", + "The device network protocol is well-established and currently used by Google" + ] + }, + { + "question": "Does the network protocol assure server-to-client identity verification?", + "type": "select", + "answer": "Yes", + "options": [ + "Yes", + "No", + "I don't know" + ], + "validation": { + "required": true + } + }, + { + "question": "Click the statements that best describe the characteristics of this device.", + "description": "This tells us about how this device is managed remotely.", + "type": "select-multiple", + "answer": [ + 0, + 1, + 2 + ], + "options": [ + "PII/PHI, or confidential business data is accessible from the device without authentication", + "Unrecoverable actions (e.g. disk wipe) can be performed remotely", + "Authentication is required for remote access", + "The management interface is accessible from the public internet", + "Static credentials are used for administration" + ] + }, + { + "question": "Are any of the following statements true about this device?", + "description": "This informs us about what other systems and processes this device is a part of.", + "type": "select-multiple", + "answer": [ + 2, + 3 + ], + "options": [ + "The device monitors an environment for active risks to human life.", + "The device is used to convey people, or critical property.", + "The device controls robotics in human-accessible spaces.", + "The device controls physical access systems.", + "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", + "The device's failure would cause faults in other high-criticality processes." + ] + } + ] +} \ No newline at end of file From b8b97e9bb5ccd5bf805a5ed60a47ccb7be7db091 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Tue, 6 Aug 2024 17:03:59 +0100 Subject: [PATCH 21/30] removed the tests with mock response --- testing/api/test_api.py | 67 ----------------------------------------- 1 file changed, 67 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 298389f76..523f89802 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -28,20 +28,7 @@ from typing import Iterator import pytest import requests -import responses -#import sys - -# from unittest.mock import patch, MagicMock - -# # Get the directory of the current script -# current_dir = os.path.dirname(os.path.abspath(__file__)) - -# # Define the relative path to the desired directory -# relative_path = os.path.join(current_dir, '../../framework/python/src') - -# # Append the relative path to sys.path -# sys.path.append(relative_path) ALL_DEVICES = "*" API = "http://127.0.0.1:8000" @@ -736,60 +723,6 @@ def test_system_latest_version(testrun): # pylint: disable=W0613 # Check if an update is available assert response["update_available"] is False -@responses.activate -def test_system_update_available(testrun): # pylint: disable=W0613 - """Test for testrun version when update is available""" - - # Mock the API response when the update is available - responses.add( - responses.GET, - f"{API}/system/version", - json={ - "installed_version": "3.1", - "update_available": True, - "latest_version": "3.1.1", - "latest_version_url": - ("https://github.com/google/testrun/releases/tag/v3.1.1") - }, - status=200 - ) - - # Send the get request to the API - r = requests.get(f"{API}/system/version", timeout=5) - - # Parse the response - response = r.json() - - # Check if status code is 200 (update available) - assert r.status_code == 200 - # Check if an update is available - assert response["update_available"] is True - -@responses.activate -def test_system_version_cannot_be_obtained(testrun): # pylint: disable=W0613 - """Test when the current version cannot be obtained""" - - # Mock the API response when the current version cannot be obtained - responses.add( - responses.GET, - f"{API}/system/version", - json={"error": "Could not fetch current version"}, - status=500 - ) - - # Send the get request to the API - r = requests.get(f"{API}/system/version", timeout=5) - - # Parse the JSON response - response = r.json() - - - # Check if the response status code is 500 - assert r.status_code == 500 - - # Check if the error in response - assert "error" in response - # Tests for reports endpoints def test_get_reports_no_reports(testrun): # pylint: disable=W0613 From 99644b72270ea7a993c994b09246dd03a4931cf6 Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Tue, 6 Aug 2024 14:48:39 -0600 Subject: [PATCH 22/30] Update NTP report (#666) * Update NTP report * cleanup imports * pylint updates --- modules/test/ntp/python/src/ntp_module.py | 76 +- .../unit/ntp/reports/ntp_report_local.html | 1398 +---------------- 2 files changed, 74 insertions(+), 1400 deletions(-) diff --git a/modules/test/ntp/python/src/ntp_module.py b/modules/test/ntp/python/src/ntp_module.py index 033e98974..be27abbad 100644 --- a/modules/test/ntp/python/src/ntp_module.py +++ b/modules/test/ntp/python/src/ntp_module.py @@ -14,8 +14,8 @@ """NTP test module""" from test_module import TestModule from scapy.all import rdpcap, IP, IPv6, NTP, UDP, Ether -from datetime import datetime import os +from collections import defaultdict LOG_NAME = 'test_ntp' MODULE_REPORT_FILE_NAME = 'ntp_report.html' @@ -69,6 +69,33 @@ def generate_module_report(self): total_responses = sum(1 for row in ntp_table_data if row['Type'] == 'Server') + # Initialize a dictionary to store timestamps for each unique combination + timestamps = defaultdict(list) + + # Collect timestamps for each unique combination + for row in ntp_table_data: + # Add the timestamp to the corresponding combination + key = (row['Source'], row['Destination'], row['Type'], row['Version']) + timestamps[key].append(row['Timestamp']) + + # Calculate the average time between requests for each unique combination + average_time_between_requests = {} + + for key, times in timestamps.items(): + # Sort the timestamps + times.sort() + + # Calculate the time differences between consecutive timestamps + time_diffs = [t2 - t1 for t1, t2 in zip(times[:-1], times[1:])] + + # Calculate the average of the time differences + if time_diffs: + avg_diff = sum(time_diffs) / len(time_diffs) + else: + avg_diff = 0 # one timestamp, the average difference is 0 + + average_time_between_requests[key] = avg_diff + # Add summary table html_content += (f''' @@ -92,7 +119,6 @@ def generate_module_report(self): ''') if total_requests + total_responses > 0: - table_content = '''
@@ -101,37 +127,39 @@ def generate_module_report(self): - + + ''' - for row in ntp_table_data: - - # Timestamp of the NTP packet - dt_object = datetime.utcfromtimestamp(row['Timestamp']) + # Generate the HTML table with the count column + for (src, dst, typ, + version), avg_diff in average_time_between_requests.items(): + cnt = len(timestamps[(src, dst, typ, version)]) - # Extract milliseconds from the fractional part of the timestamp - milliseconds = int((row['Timestamp'] % 1) * 1000) - - # Format the datetime object with milliseconds - formatted_time = dt_object.strftime( - '%b %d, %Y %H:%M:%S.') + f'{milliseconds:03d}' + # Sync Average only applies to client requests + if 'Client' in typ: + # Convert avg_diff to seconds and format it + avg_diff_seconds = avg_diff + avg_formatted_time = f'{avg_diff_seconds:.3f} seconds' + else: + avg_formatted_time = 'N/A' - table_content += (f''' + table_content += f''' - - - - - - ''') + + + + + + + ''' table_content += '''
Destination Type VersionTimestampCountSync Request Average
{row['Source']}{row['Destination']}{row['Type']}{row['Version']}{formatted_time}
{src}{dst}{typ}{version}{cnt}{avg_formatted_time}
''' - html_content += table_content else: @@ -159,8 +187,8 @@ def extract_ntp_data(self): # Read the pcap files packets = (rdpcap(self.startup_capture_file) + - rdpcap(self.monitor_capture_file) + - rdpcap(self.ntp_server_capture_file)) + rdpcap(self.monitor_capture_file) + + rdpcap(self.ntp_server_capture_file)) # Iterate through NTP packets for packet in packets: @@ -283,7 +311,7 @@ def _ntp_network_ntp_dhcp(self): 'server and non-DHCP provided server') elif ntp_to_remote: result = ('Feature Not Detected', - 'Device sent NTP request to non-DHCP provided server') + 'Device sent NTP request to non-DHCP provided server') elif ntp_to_local: result = True, 'Device sent NTP request to DHCP provided server' diff --git a/testing/unit/ntp/reports/ntp_report_local.html b/testing/unit/ntp/reports/ntp_report_local.html index a08c42f9d..c9715fba5 100644 --- a/testing/unit/ntp/reports/ntp_report_local.html +++ b/testing/unit/ntp/reports/ntp_report_local.html @@ -25,7 +25,8 @@

NTP Module

Destination Type Version - Timestamp + Count + Sync Request Average @@ -34,1435 +35,80 @@

NTP Module

216.239.35.12 Client 4 - Feb 15, 2024 22:12:28.681 + 8 + 37.942 seconds 216.239.35.12 10.10.10.15 Server 4 - Feb 15, 2024 22:12:28.728 + 8 + N/A 10.10.10.15 216.239.35.4 Client 4 - Feb 15, 2024 22:12:28.842 + 8 + 37.834 seconds 216.239.35.4 10.10.10.15 Server 4 - Feb 15, 2024 22:12:28.888 + 8 + N/A 10.10.10.15 216.239.35.8 Client 4 - Feb 15, 2024 22:12:29.042 + 8 + 38.056 seconds 216.239.35.8 10.10.10.15 Server 4 - Feb 15, 2024 22:12:29.089 + 8 + N/A 10.10.10.15 216.239.35.0 Client 4 - Feb 15, 2024 22:12:29.243 + 14 + 20.601 seconds 216.239.35.0 10.10.10.15 Server 4 - Feb 15, 2024 22:12:29.290 + 17 + N/A 10.10.10.15 10.10.10.5 Client 4 - Feb 15, 2024 22:12:29.447 + 63 + 13.057 seconds 10.10.10.5 10.10.10.15 Server 4 - Feb 15, 2024 22:12:29.448 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:12:30.802 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:30.850 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:12:30.973 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.032 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:12:31.173 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.220 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:12:31.376 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.423 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:12:32.867 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:32.914 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:12:33.112 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.159 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:12:33.271 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.318 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:12:33.475 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.522 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:12:34.956 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.002 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:12:35.182 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.228 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:12:35.398 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.445 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:12:35.625 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.673 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:35.785 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.786 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:41.931 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:41.932 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:43.954 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:43.956 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:06.439 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:06.489 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:08.492 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:08.494 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:08.543 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:13:40.310 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.357 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:13:40.512 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:40.536 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.542 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.574 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.583 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:13:40.714 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.764 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:13:40.917 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.965 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:48.274 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:48.277 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:12.619 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:12.624 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:12.668 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:14:44.515 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:44.562 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:44.702 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:44.704 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:14:45.158 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:45.219 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:14:45.359 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:45.406 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:14:45.707 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:45.755 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:14:45.980 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:46.027 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:53.026 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:53.029 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:16.786 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:16.791 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:15:18.794 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:18.843 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:48.884 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:48.887 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:15:49.063 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:49.110 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:15:49.462 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:49.509 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:15:50.127 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:50.175 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:15:51.107 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:51.154 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:15:51.890 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:51.938 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:20.970 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:20.971 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:16:24.975 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:25.023 - - - 10.10.10.15 - 216.239.35.4 - Client - 4 - Feb 15, 2024 22:16:53.677 - - - 216.239.35.4 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:53.739 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.15 - 216.239.35.12 - Client - 4 - Feb 15, 2024 22:16:54.276 - - - 216.239.35.12 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.322 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:16:54.593 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.648 - - - 10.10.10.15 - 216.239.35.8 - Client - 4 - Feb 15, 2024 22:16:55.435 - - - 216.239.35.8 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:55.481 - - - 10.10.10.15 - 216.239.35.0 - Client - 4 - Feb 15, 2024 22:16:57.059 - - - 216.239.35.0 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:57.107 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:02.738 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:02.740 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:26.136 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:26.139 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:29.447 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:29.448 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:31.577 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:33.694 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:35.785 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:35.786 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:37.806 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:39.856 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:41.931 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:41.932 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:12:43.954 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:12:43.956 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:06.439 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:08.492 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:08.494 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:40.536 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:40.541 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:13:48.274 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:13:48.277 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:12.619 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:12.624 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:44.702 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:44.703 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:14:53.026 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:14:53.029 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:16.786 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:16.791 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:48.884 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:48.887 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:15:57.829 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:20.970 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:20.970 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:16:54.054 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:02.738 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:02.740 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:26.136 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:26.139 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:17:59.293 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:17:59.293 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:18:07.242 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:18:07.242 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:18:32.379 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:18:32.379 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:06.908 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:06.908 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:08.936 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:08.937 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:10.974 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:10.974 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:12.998 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:12.999 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:20:59.581 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:20:59.582 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:34.063 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:34.063 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:36.121 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:36.121 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:38.176 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:38.176 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:21:40.277 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:21:40.277 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:22:05.704 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:22:05.706 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:22:45.469 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:22:45.470 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:23:09.826 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:23:09.828 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:23:50.337 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:23:50.343 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:24:13.945 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:24:13.946 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:24:54.876 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:24:54.877 - - - 10.10.10.15 - 10.10.10.5 - Client - 4 - Feb 15, 2024 22:25:59.000 - - - 10.10.10.5 - 10.10.10.15 - Server - 4 - Feb 15, 2024 22:25:59.001 + 63 + N/A From ab055daf2cf1c81bc1cfcdf6482aa734f73d1742 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Wed, 7 Aug 2024 17:02:28 +0100 Subject: [PATCH 23/30] modified update profile for bad request --- framework/python/src/api/api.py | 13 +- framework/python/src/common/session.py | 74 +++++---- testing/api/profiles/no_answer.json | 143 ----------------- .../api/profiles/profile_invalid_format.json | 144 ------------------ testing/api/test_api.py | 41 +---- 5 files changed, 55 insertions(+), 360 deletions(-) delete mode 100644 testing/api/profiles/no_answer.json delete mode 100644 testing/api/profiles/profile_invalid_format.json diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 5547420ac..279539613 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -702,19 +702,12 @@ async def update_profile(self, request: Request, response: Response): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid request received") - profile_name = req_json.get("name") - - # Error handling if profile name not in request - if not profile_name: - LOGGER.error("Missing 'name' in the request JSON") + # Validate json profile + if not self._session.validate_profile_json(req_json): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid request received") - # Error handling if 'questions' not in request - if "questions" not in req_json: - LOGGER.error("Missing 'questions' in the request JSON") - response.status_code = status.HTTP_400_BAD_REQUEST - return self._generate_msg(False, "Invalid request received") + profile_name = req_json.get("name") # Check if profile exists profile = self.get_session().get_profile(profile_name) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 02d46591e..fc0afe3f3 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -466,44 +466,63 @@ def _get_profile_question(self, profile_json, question): return None - def update_profile(self, profile_json): + # Validate the JSON profile for completeness + def validate_profile_json(self, profile_json): - profile_name = profile_json['name'] + # Check if 'name' exists in profile + if 'name' not in profile_json: + LOGGER.error("Missing 'name' in profile") + return False - # Add version, timestamp and status - profile_json['version'] = self.get_version() - profile_json['created'] = datetime.datetime.now().strftime('%Y-%m-%d') + # Check if 'name' field not empty + elif len(profile_json.get('name')) == 0: + LOGGER.error('Name field left empty') + return False - if 'status' in profile_json and profile_json.get('status') == 'Valid': - # Attempting to submit a risk profile, we need to check it + # Error handling if 'questions' not in request + if 'questions' not in profile_json: + LOGGER.error("Missing 'questions' field in profile") + return False - # Check all questions have been answered - all_questions_answered = True + # Error handling if 'question' is missing + for question in profile_json.get('questions'): + if 'question' not in question: + LOGGER.error("The 'question' field is missing") + return False - for question in self.get_profiles_format(): + # Check if 'question' field not empty + elif len(question.get('question')) == 0: + LOGGER.error("A question is missing from 'question' field") + return False - # Check question is present - profile_question = self._get_profile_question(profile_json, - question.get('question')) + # Error handling if 'answer' is missing + for answer in profile_json.get('questions'): + if 'answer' not in answer: + LOGGER.error('The answer field is missing') + return False + + # Attempting to submit a valid risk profile + if 'status' in profile_json and profile_json.get('status') == 'Valid': + # Error handling if 'answer' is missing + for answer in profile_json.get('questions'): + if len(answer.get('answer')) == 0: + LOGGER.error("The 'answer' field is missing") + return False - if profile_question is not None: + return True - # Check answer is present - if 'answer' not in profile_question: - LOGGER.error('Missing answer for question: ' + - question.get('question')) - all_questions_answered = False + def update_profile(self, profile_json): - else: - LOGGER.error('Missing question: ' + question.get('question')) - all_questions_answered = False + profile_name = profile_json['name'] + + # Add version, timestamp and status + profile_json['version'] = self.get_version() + profile_json['created'] = datetime.datetime.now().strftime('%Y-%m-%d') - if not all_questions_answered: - LOGGER.error('Not all questions answered') - return None + # Validate profile data + if not self.validate_profile_json(profile_json): + return None - else: - profile_json['status'] = 'Draft' risk_profile = self.get_profile(profile_name) @@ -536,6 +555,7 @@ def update_profile(self, profile_json): return risk_profile def delete_profile(self, profile): + print(f"delete_profile called with profile: {profile}") try: profile_name = profile.name diff --git a/testing/api/profiles/no_answer.json b/testing/api/profiles/no_answer.json deleted file mode 100644 index 9f3c033d3..000000000 --- a/testing/api/profiles/no_answer.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "name": "New Profile", - "status": "Valid", - "created": "2024-05-23 12:38:26", - "version": "v1.3", - "questions": [ - { - "question": "What type of device is this?", - "type": "select", - "options": [ - "IoT Sensor", - "IoT Controller", - "Smart Device", - "Something else" - ], - "validation": { - "required": true - } - }, - { - "question": "How will this device be used at Google?", - "type": "text-long", - "validation": { - "max": "128", - "required": true - } - }, - { - "question": "What is the email of the device owner(s)?", - "type": "email-multiple", - "answer": "boddey@google.com, cmeredith@google.com", - "validation": { - "required": true, - "max": "128" - } - }, - { - "question": "Is this device going to be managed by Google or a third party?", - "type": "select", - "options": [ - "Google", - "Third Party" - ], - "answer": "Google", - "validation": { - "required": true - } - }, - { - "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", - "type": "select", - "options": [ - "Yes", - "No", - "N/A" - ], - "default": "N/A", - "answer": "Yes", - "validation": { - "required": true - } - }, - { - "question": "Are any of the following statements true about your device?", - "description": "This tells us about the data your device will collect", - "type": "select-multiple", - "answer": [ - 0, - 2 - ], - "options": [ - "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", - "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", - "The device stream confidential business data in real-time (seconds)?" - ] - }, - { - "question": "Which of the following statements are true about this device?", - "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", - "type": "select-multiple", - "answer": [ - 0, - 1, - 5 - ], - "options": [ - "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", - "Data transmission occurs across less-trusted networks (e.g. the internet).", - "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", - "A confidentiality breach during transmission would have a substantial negative impact", - "The device encrypts data during transmission", - "The device network protocol is well-established and currently used by Google" - ] - }, - { - "question": "Does the network protocol assure server-to-client identity verification?", - "type": "select", - "answer": "Yes", - "options": [ - "Yes", - "No", - "I don't know" - ], - "validation": { - "required": true - } - }, - { - "question": "Click the statements that best describe the characteristics of this device.", - "description": "This tells us about how this device is managed remotely.", - "type": "select-multiple", - "answer": [ - 0, - 1, - 2 - ], - "options": [ - "PII/PHI, or confidential business data is accessible from the device without authentication", - "Unrecoverable actions (e.g. disk wipe) can be performed remotely", - "Authentication is required for remote access", - "The management interface is accessible from the public internet", - "Static credentials are used for administration" - ] - }, - { - "question": "Are any of the following statements true about this device?", - "description": "This informs us about what other systems and processes this device is a part of.", - "type": "select-multiple", - "answer": [ - 2, - 3 - ], - "options": [ - "The device monitors an environment for active risks to human life.", - "The device is used to convey people, or critical property.", - "The device controls robotics in human-accessible spaces.", - "The device controls physical access systems.", - "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", - "The device's failure would cause faults in other high-criticality processes." - ] - } - ] -} \ No newline at end of file diff --git a/testing/api/profiles/profile_invalid_format.json b/testing/api/profiles/profile_invalid_format.json deleted file mode 100644 index a88e8e6b7..000000000 --- a/testing/api/profiles/profile_invalid_format.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "status": "Draft", - "created": "2024-05-23 12:38:26", - "version": "v1.3", - "questions": [ - { - "question": "What type of device is this?", - "type": "select", - "options": [ - "IoT Sensor", - "IoT Controller", - "Smart Device", - "Something else" - ], - "answer": "IoT Sensor", - "validation": { - "required": true - } - }, - { - "question": "How will this device be used at Google?", - "type": "text-long", - "answer": "Installed in a building", - "validation": { - "max": "128", - "required": true - } - }, - { - "question": "What is the email of the device owner(s)?", - "type": "email-multiple", - "answer": "boddey@google.com, cmeredith@google.com", - "validation": { - "required": true, - "max": "128" - } - }, - { - "question": "Is this device going to be managed by Google or a third party?", - "type": "select", - "options": [ - "Google", - "Third Party" - ], - "answer": "Google", - "validation": { - "required": true - } - }, - { - "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", - "type": "select", - "options": [ - "Yes", - "No", - "N/A" - ], - "default": "N/A", - "answer": "Yes", - "validation": { - "required": true - } - }, - { - "question": "Are any of the following statements true about your device?", - "description": "This tells us about the data your device will collect", - "type": "select-multiple", - "answer": [ - 0, - 2 - ], - "options": [ - "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", - "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", - "The device stream confidential business data in real-time (seconds)?" - ] - }, - { - "question": "Which of the following statements are true about this device?", - "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", - "type": "select-multiple", - "answer": [ - 0, - 1, - 5 - ], - "options": [ - "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", - "Data transmission occurs across less-trusted networks (e.g. the internet).", - "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", - "A confidentiality breach during transmission would have a substantial negative impact", - "The device encrypts data during transmission", - "The device network protocol is well-established and currently used by Google" - ] - }, - { - "question": "Does the network protocol assure server-to-client identity verification?", - "type": "select", - "answer": "Yes", - "options": [ - "Yes", - "No", - "I don't know" - ], - "validation": { - "required": true - } - }, - { - "question": "Click the statements that best describe the characteristics of this device.", - "description": "This tells us about how this device is managed remotely.", - "type": "select-multiple", - "answer": [ - 0, - 1, - 2 - ], - "options": [ - "PII/PHI, or confidential business data is accessible from the device without authentication", - "Unrecoverable actions (e.g. disk wipe) can be performed remotely", - "Authentication is required for remote access", - "The management interface is accessible from the public internet", - "Static credentials are used for administration" - ] - }, - { - "question": "Are any of the following statements true about this device?", - "description": "This informs us about what other systems and processes this device is a part of.", - "type": "select-multiple", - "answer": [ - 2, - 3 - ], - "options": [ - "The device monitors an environment for active risks to human life.", - "The device is used to convey people, or critical property.", - "The device controls robotics in human-accessible spaces.", - "The device controls physical access systems.", - "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", - "The device's failure would cause faults in other high-criticality processes." - ] - } - ] -} \ No newline at end of file diff --git a/testing/api/test_api.py b/testing/api/test_api.py index 523f89802..c5f78d5e8 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -743,18 +743,6 @@ def test_get_reports_no_reports(testrun): # pylint: disable=W0613 # Check if the response is an empty list assert response == [] -# def test_delete_report(testrun): # pylint: disable=W0613 -# """Test the delete report endpoint""" -# pass - -# def test_get_report(testrun): # pylint: disable=W0613 -# """Test the get report endpoint""" -# pass - -# def test_export_report(testrun): # pylint: disable=W0613 - # """Test the export report endpoint""" - # pass - # Tests for device endpoints @pytest.mark.skip() @@ -1694,9 +1682,8 @@ def test_update_profile_invalid_json(testrun, reset_profiles, add_profile): # py # Load the new profile using add_profile fixture add_profile("new_profile.json") - # Load the updated profile using load_json utility method - updated_profile = load_json("profile_invalid_format.json", - directory="testing/api/profiles") + # invalid JSON + updated_profile = {} # Send the post request to update the profile r = requests.post( @@ -1716,9 +1703,8 @@ def test_update_profile_invalid_json(testrun, reset_profiles, add_profile): # py def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable=W0613 """Test for create profile invalid JSON payload """ - # Load the profile using load_json utility method - new_profile = load_json("profile_invalid_format.json", - directory="testing/api/profiles") + # invalid JSON + new_profile = {} # Send the post request to update the profile r = requests.post( @@ -1753,21 +1739,6 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response -def test_create_update_profile_missing_answer(testrun): # pylint: disable=W0613 - """Test for create/update profile causing bad request.""" - - # Load the json file - profile = load_json("no_answer.json", directory="testing/api/profiles") - - # Send teh post request - r = requests.post(f"{API}/profiles", data=json.dumps(profile), timeout=5) - - # Parse the json response - print(r.json()) - - # Check if status code is 400 (bad request) - assert r.status_code == 400 - def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for delete profile""" @@ -1844,9 +1815,7 @@ def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response - profile_to_delete_2 = load_json("profile_invalid_format.json", - directory="testing/api/profiles") - + profile_to_delete_2 = {"status": "Draft"} # Delete the profile r = requests.delete( f"{API}/profiles", From 9c60f6195b201354609bb9eb5cd3ec13ef8d8bf8 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Thu, 8 Aug 2024 10:06:24 +0100 Subject: [PATCH 24/30] changed validate_profile_json: handling empty spaces in name and question, handle if 'risk' field missing if status is 'Valid' --- framework/python/src/common/session.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index fc0afe3f3..81477e572 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -475,7 +475,7 @@ def validate_profile_json(self, profile_json): return False # Check if 'name' field not empty - elif len(profile_json.get('name')) == 0: + elif len(profile_json.get('name').strip()) == 0: LOGGER.error('Name field left empty') return False @@ -491,7 +491,7 @@ def validate_profile_json(self, profile_json): return False # Check if 'question' field not empty - elif len(question.get('question')) == 0: + elif len(question.get('question').strip()) == 0: LOGGER.error("A question is missing from 'question' field") return False @@ -503,11 +503,11 @@ def validate_profile_json(self, profile_json): # Attempting to submit a valid risk profile if 'status' in profile_json and profile_json.get('status') == 'Valid': - # Error handling if 'answer' is missing - for answer in profile_json.get('questions'): - if len(answer.get('answer')) == 0: - LOGGER.error("The 'answer' field is missing") - return False + + # Check if 'risk' exists in profile + if 'risk' not in profile_json: + LOGGER.error("Missing 'risk' in profile") + return False return True From 04081304756a6e1d331484ff621ee880e9c73713 Mon Sep 17 00:00:00 2001 From: MariusBalovin Date: Thu, 8 Aug 2024 16:03:13 +0100 Subject: [PATCH 25/30] updated the requested changes --- testing/api/test_api.py | 40 +++++----------------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/testing/api/test_api.py b/testing/api/test_api.py index c5f78d5e8..c59f6eba1 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -40,6 +40,7 @@ PROFILES_DIRECTORY = "local/risk_profiles" SYSTEM_CONFIG_PATH = "local/system.json" SYSTEM_CONFIG_RESTORE_PATH = "testing/api/system.json" +PROFILES_PATH = "testing/api/profiles" BASELINE_MAC_ADDR = "02:42:aa:00:01:01" ALL_MAC_ADDR = "02:42:aa:00:00:01" @@ -1391,20 +1392,6 @@ def test_get_test_modules(testrun): # pylint: disable=W0613 # Check if the response is a list assert isinstance(response, list) - # Define the expected modules - expected_modules = [ - "Connection", - "Services", - "NTP", - "DNS", - "Protocol", - "TLS" - ] - - # Check if all expected modules are in the response - for module in expected_modules: - assert module in response - # Tests for profile endpoints def delete_all_profiles(): """Utility method to delete all profiles from risk_profiles folder""" @@ -1436,7 +1423,7 @@ def create_profile(file_name): """Utility method to create the profile""" # Load the profile - new_profile = load_json(file_name, directory="testing/api/profiles") + new_profile = load_json(file_name, directory=PROFILES_PATH) # Assign the profile name to profile_name profile_name = new_profile["name"] @@ -1586,7 +1573,7 @@ def test_create_profile(testrun, reset_profiles): # pylint: disable=W0613 """Test for create profile if not exists""" # Load the profile - new_profile = load_json("new_profile.json", directory="testing/api/profiles") + new_profile = load_json("new_profile.json", directory=PROFILES_PATH) # Assign the profile name to profile_name profile_name = new_profile["name"] @@ -1632,7 +1619,7 @@ def test_update_profile(testrun, reset_profiles, add_profile): # pylint: disable # Load the updated profile using load_json utility method updated_profile = load_json("updated_profile.json", - directory="testing/api/profiles") + directory=PROFILES_PATH) # Assign the new_profile name profile_name = new_profile["name"] @@ -1721,24 +1708,6 @@ def test_create_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if "error" key in response assert "error" in response - # Load the 2nd profile - new_profile_2 = {"name": "New Profile"} - - # Send the post request to update the profile - r = requests.post( - f"{API}/profiles", - data=json.dumps(new_profile_2), - timeout=5) - - # Parse the response - response = r.json() - - # Check if status code is 400 (Bad request) - assert r.status_code == 400 - - # Check if "error" key in response - assert "error" in response - def test_delete_profile(testrun, reset_profiles, add_profile): # pylint: disable=W0613 """Test for delete profile""" @@ -1827,6 +1796,7 @@ def test_delete_profile_invalid_json(testrun, reset_profiles): # pylint: disable # Check if status code is 400 (bad request) assert r.status_code == 400 + # Check if "error" key in response assert "error" in response From f391c6ff02be4097d970747fab4f22506dd9f451 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 8 Aug 2024 16:22:54 +0100 Subject: [PATCH 26/30] Add further profile validation --- framework/python/src/api/api.py | 104 ++++++++++++++++++++++++- framework/python/src/common/session.py | 79 ++----------------- 2 files changed, 111 insertions(+), 72 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 279539613..28e25117c 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -703,7 +703,7 @@ async def update_profile(self, request: Request, response: Response): return self._generate_msg(False, "Invalid request received") # Validate json profile - if not self._session.validate_profile_json(req_json): + if not self._validate_profile_json(req_json): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid request received") @@ -774,6 +774,108 @@ async def delete_profile(self, request: Request, response: Response): return self._generate_msg(True, "Successfully deleted that profile") + def _validate_profile_json(self, profile_json): + """Validate properties in profile update requests""" + + profile_format = self.get_session().get_profiles_format() + + # Get the status field + valid = False + if "status" in profile_json and profile_json.get("status") == "Valid": + valid = True + + # Check if "name" exists in profile + if "name" not in profile_json: + LOGGER.error("Missing 'name' in profile") + return False + + # Check if "name" field not empty + elif len(profile_json.get("name").strip()) == 0: + LOGGER.error("Name field left empty") + return False + + # Error handling if "questions" not in request + if "questions" not in profile_json and valid: + LOGGER.error("Missing 'questions' field in profile") + return False + + # Validating the questions section + for question in profile_json.get("questions"): + + # Check if the question field is present + if "question" not in question: + LOGGER.error("The 'question' field is missing") + return False + + # Check if "question" field not empty + elif len(question.get("question").strip()) == 0: + LOGGER.error("A question is missing from 'question' field") + return False + + # Check if question is a recognized question + format_q = self.get_session().get_profile_format_question( + question.get("question")) + + if format_q is None: + LOGGER.error(f"Unrecognized question: {question.get('question')}") + return False + + # Error handling if "answer" is missing + if "answer" not in question and valid: + LOGGER.error("The answer field is missing") + return False + + # If answer is present, check the validation rules + else: + + # Extract the answer out of the profile + answer = question.get("answer") + + # Get the validation rules + field_type = format_q.get("type") + + # Check if type is string or single select, answer should be a string + if ((field_type == "string" or field_type == "select") + and not isinstance(answer, str)): + LOGGER.error(f"""Answer for question \ +{question.get('question')} is incorrect data type""") + return False + + # Check if type is select, answer must be from list + if field_type == "select": + possible_answers = format_q.get("options") + if answer not in possible_answers: + LOGGER.error(f"""Answer for question \ +{question.get('question')} is not valid""") + return False + + # Validate select multiple field types + if field_type == "select-multiple": + + if not isinstance(answer, list): + LOGGER.error(f"""Answer for question \ +{question.get('question')} is incorrect data type""") + return False + + question_options_len = len(format_q.get("options")) + + # We know it is a list, now check the indexes + for index in answer: + + # Check if the index is an integer + if not isinstance(index, int): + LOGGER.error(f"""Answer for question \ +{question.get('question')} is incorrect data type""") + return False + + # Check if index is 0 or above and less than the num of options + if index < 0 or index >= question_options_len: + LOGGER.error(f"""Invalid index provided as answer for \ +question {question.get('question')}""") + return False + + return True + # Certificates def get_certs(self): LOGGER.debug("Received certs list request") diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 81477e572..d07f16dbc 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -45,7 +45,7 @@ class TestrunSession(): - """Represents the current session of Test Run.""" + """Represents the current session of Testrun.""" def __init__(self, root_dir): self._root_dir = root_dir @@ -439,25 +439,6 @@ def get_profile(self, name): return profile return None - def validate_profile(self, profile_json): - - # Check name field is present - if 'name' not in profile_json: - return False - - # Check questions field is present - if 'questions' not in profile_json: - return False - - # Check all questions are present - for format_q in self.get_profiles_format(): - if self._get_profile_question(profile_json, - format_q.get('question')) is None: - LOGGER.error('Missing question: ' + format_q.get('question')) - return False - - return True - def _get_profile_question(self, profile_json, question): for q in profile_json.get('questions'): @@ -466,52 +447,14 @@ def _get_profile_question(self, profile_json, question): return None - # Validate the JSON profile for completeness - def validate_profile_json(self, profile_json): - - # Check if 'name' exists in profile - if 'name' not in profile_json: - LOGGER.error("Missing 'name' in profile") - return False - - # Check if 'name' field not empty - elif len(profile_json.get('name').strip()) == 0: - LOGGER.error('Name field left empty') - return False - - # Error handling if 'questions' not in request - if 'questions' not in profile_json: - LOGGER.error("Missing 'questions' field in profile") - return False - - # Error handling if 'question' is missing - for question in profile_json.get('questions'): - if 'question' not in question: - LOGGER.error("The 'question' field is missing") - return False - - # Check if 'question' field not empty - elif len(question.get('question').strip()) == 0: - LOGGER.error("A question is missing from 'question' field") - return False - - # Error handling if 'answer' is missing - for answer in profile_json.get('questions'): - if 'answer' not in answer: - LOGGER.error('The answer field is missing') - return False - - # Attempting to submit a valid risk profile - if 'status' in profile_json and profile_json.get('status') == 'Valid': - - # Check if 'risk' exists in profile - if 'risk' not in profile_json: - LOGGER.error("Missing 'risk' in profile") - return False - - return True + def get_profile_format_question(self, question): + for q in self.get_profiles_format(): + if q.get('question') == question: + return q def update_profile(self, profile_json): + """Update the risk profile with the provided JSON. + The content has already been validated in the API""" profile_name = profile_json['name'] @@ -519,13 +462,8 @@ def update_profile(self, profile_json): profile_json['version'] = self.get_version() profile_json['created'] = datetime.datetime.now().strftime('%Y-%m-%d') - # Validate profile data - if not self.validate_profile_json(profile_json): - return None - - + # Check if profile already exists risk_profile = self.get_profile(profile_name) - if risk_profile is None: # Create a new risk profile @@ -555,7 +493,6 @@ def update_profile(self, profile_json): return risk_profile def delete_profile(self, profile): - print(f"delete_profile called with profile: {profile}") try: profile_name = profile.name From 6844da14b04e6656bf556aee76594599419eb679 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 8 Aug 2024 16:26:56 +0100 Subject: [PATCH 27/30] Fix pylint issues --- framework/python/src/api/api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 28e25117c..076c0066a 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -777,8 +777,6 @@ async def delete_profile(self, request: Request, response: Response): def _validate_profile_json(self, profile_json): """Validate properties in profile update requests""" - profile_format = self.get_session().get_profiles_format() - # Get the status field valid = False if "status" in profile_json and profile_json.get("status") == "Valid": @@ -835,7 +833,7 @@ def _validate_profile_json(self, profile_json): field_type = format_q.get("type") # Check if type is string or single select, answer should be a string - if ((field_type == "string" or field_type == "select") + if ((field_type in ["string", "select"]) and not isinstance(answer, str)): LOGGER.error(f"""Answer for question \ {question.get('question')} is incorrect data type""") From c4c1803df9a5a821e7c4b4cc945c5538ce9b5ba0 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Thu, 8 Aug 2024 17:33:27 +0100 Subject: [PATCH 28/30] Fix profile tests --- testing/api/profiles/new_profile.json | 191 ++++++---------------- testing/api/profiles/new_profile_2.json | 99 +---------- testing/api/profiles/updated_profile.json | 99 +---------- testing/api/test_api.py | 28 ++-- 4 files changed, 75 insertions(+), 342 deletions(-) diff --git a/testing/api/profiles/new_profile.json b/testing/api/profiles/new_profile.json index 7043f6cfa..d63ecd17c 100644 --- a/testing/api/profiles/new_profile.json +++ b/testing/api/profiles/new_profile.json @@ -1,145 +1,54 @@ { "name": "New Profile", - "status": "Draft", - "created": "2024-05-23 12:38:26", - "version": "v1.3", + "status": "Valid", "questions": [ - { - "question": "What type of device is this?", - "type": "select", - "options": [ - "IoT Sensor", - "IoT Controller", - "Smart Device", - "Something else" - ], - "answer": "IoT Sensor", - "validation": { - "required": true - } - }, - { - "question": "How will this device be used at Google?", - "type": "text-long", - "answer": "Installed in a building", - "validation": { - "max": "128", - "required": true - } - }, - { - "question": "What is the email of the device owner(s)?", - "type": "email-multiple", - "answer": "boddey@google.com, cmeredith@google.com", - "validation": { - "required": true, - "max": "128" - } - }, - { - "question": "Is this device going to be managed by Google or a third party?", - "type": "select", - "options": [ - "Google", - "Third Party" - ], - "answer": "Google", - "validation": { - "required": true - } - }, - { - "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", - "type": "select", - "options": [ - "Yes", - "No", - "N/A" - ], - "default": "N/A", - "answer": "Yes", - "validation": { - "required": true - } - }, - { - "question": "Are any of the following statements true about your device?", - "description": "This tells us about the data your device will collect", - "type": "select-multiple", - "answer": [ - 0, - 2 - ], - "options": [ - "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", - "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", - "The device stream confidential business data in real-time (seconds)?" - ] - }, - { - "question": "Which of the following statements are true about this device?", - "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", - "type": "select-multiple", - "answer": [ - 0, - 1, - 5 - ], - "options": [ - "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", - "Data transmission occurs across less-trusted networks (e.g. the internet).", - "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", - "A confidentiality breach during transmission would have a substantial negative impact", - "The device encrypts data during transmission", - "The device network protocol is well-established and currently used by Google" - ] - }, - { - "question": "Does the network protocol assure server-to-client identity verification?", - "type": "select", - "answer": "Yes", - "options": [ - "Yes", - "No", - "I don't know" - ], - "validation": { - "required": true - } - }, - { - "question": "Click the statements that best describe the characteristics of this device.", - "description": "This tells us about how this device is managed remotely.", - "type": "select-multiple", - "answer": [ - 0, - 1, - 2 - ], - "options": [ - "PII/PHI, or confidential business data is accessible from the device without authentication", - "Unrecoverable actions (e.g. disk wipe) can be performed remotely", - "Authentication is required for remote access", - "The management interface is accessible from the public internet", - "Static credentials are used for administration" - ] - }, - { - "question": "Are any of the following statements true about this device?", - "description": "This informs us about what other systems and processes this device is a part of.", - "type": "select-multiple", - "answer": [ - 2, - 3 - ], - "options": [ - "The device monitors an environment for active risks to human life.", - "The device is used to convey people, or critical property.", - "The device controls robotics in human-accessible spaces.", - "The device controls physical access systems.", - "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", - "The device's failure would cause faults in other high-criticality processes." - ] - } + { + "question": "What type of device is this?", + "answer": "IoT Gateway" + }, + { + "question": "How will this device be used at Google?", + "answer": "Monitoring" + }, + { + "question": "Is this device going to be managed by Google or a third party?", + "answer": "Google" + }, + { + "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", + "answer": "N/A" + }, + { + "question": "Are any of the following statements true about your device?", + "answer": [ + 0 + ] + }, + { + "question": "Which of the following statements are true about this device?", + "answer": [ + 0 + ] + }, + { + "question": "Does the network protocol assure server-to-client identity verification?", + "answer": "Yes" + }, + { + "question": "Click the statements that best describe the characteristics of this device.", + "answer": [ + 0 + ] + }, + { + "question": "Are any of the following statements true about this device?", + "answer": [ + 0 + ] + }, + { + "question": "Comments", + "answer": "" + } ] -} \ No newline at end of file + } \ No newline at end of file diff --git a/testing/api/profiles/new_profile_2.json b/testing/api/profiles/new_profile_2.json index 51782e187..2ac93dc17 100644 --- a/testing/api/profiles/new_profile_2.json +++ b/testing/api/profiles/new_profile_2.json @@ -1,144 +1,55 @@ { "name": "New Profile 2", "status": "Draft", - "created": "2024-05-23 12:38:26", - "version": "v1.3", "questions": [ { "question": "What type of device is this?", - "type": "select", - "options": [ - "IoT Sensor", - "IoT Controller", - "Smart Device", - "Something else" - ], - "answer": "IoT Sensor", - "validation": { - "required": true - } + "answer": "IoT Gateway" }, { "question": "How will this device be used at Google?", - "type": "text-long", - "answer": "Installed in a building", - "validation": { - "max": "128", - "required": true - } - }, - { - "question": "What is the email of the device owner(s)?", - "type": "email-multiple", - "answer": "boddey@google.com, cmeredith@google.com", - "validation": { - "required": true, - "max": "128" - } + "answer": "Installed in a building" }, { "question": "Is this device going to be managed by Google or a third party?", - "type": "select", - "options": [ - "Google", - "Third Party" - ], - "answer": "Google", - "validation": { - "required": true - } + "answer": "Google" }, { "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", - "type": "select", - "options": [ - "Yes", - "No", - "N/A" - ], - "default": "N/A", - "answer": "Yes", - "validation": { - "required": true - } + "answer": "Yes" }, { "question": "Are any of the following statements true about your device?", - "description": "This tells us about the data your device will collect", - "type": "select-multiple", "answer": [ 0, 2 - ], - "options": [ - "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", - "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", - "The device stream confidential business data in real-time (seconds)?" ] }, { "question": "Which of the following statements are true about this device?", - "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", - "type": "select-multiple", "answer": [ 0, 1, 5 - ], - "options": [ - "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", - "Data transmission occurs across less-trusted networks (e.g. the internet).", - "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", - "A confidentiality breach during transmission would have a substantial negative impact", - "The device encrypts data during transmission", - "The device network protocol is well-established and currently used by Google" ] }, { "question": "Does the network protocol assure server-to-client identity verification?", - "type": "select", - "answer": "Yes", - "options": [ - "Yes", - "No", - "I don't know" - ], - "validation": { - "required": true - } + "answer": "Yes" }, { "question": "Click the statements that best describe the characteristics of this device.", - "description": "This tells us about how this device is managed remotely.", - "type": "select-multiple", "answer": [ 0, 1, 2 - ], - "options": [ - "PII/PHI, or confidential business data is accessible from the device without authentication", - "Unrecoverable actions (e.g. disk wipe) can be performed remotely", - "Authentication is required for remote access", - "The management interface is accessible from the public internet", - "Static credentials are used for administration" ] }, { "question": "Are any of the following statements true about this device?", - "description": "This informs us about what other systems and processes this device is a part of.", - "type": "select-multiple", "answer": [ 2, 3 - ], - "options": [ - "The device monitors an environment for active risks to human life.", - "The device is used to convey people, or critical property.", - "The device controls robotics in human-accessible spaces.", - "The device controls physical access systems.", - "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", - "The device's failure would cause faults in other high-criticality processes." ] } ] diff --git a/testing/api/profiles/updated_profile.json b/testing/api/profiles/updated_profile.json index a659cd937..91714bcfa 100644 --- a/testing/api/profiles/updated_profile.json +++ b/testing/api/profiles/updated_profile.json @@ -2,144 +2,55 @@ "name": "New Profile", "rename": "Updated Profile", "status": "Draft", - "created": "2024-05-23 12:38:26", - "version": "v1.3", "questions": [ { "question": "What type of device is this?", - "type": "select", - "options": [ - "IoT Sensor", - "IoT Controller", - "Smart Device", - "Something else" - ], - "answer": "IoT Sensor", - "validation": { - "required": true - } + "answer": "IoT Gateway" }, { "question": "How will this device be used at Google?", - "type": "text-long", - "answer": "Installed in a building", - "validation": { - "max": "128", - "required": true - } - }, - { - "question": "What is the email of the device owner(s)?", - "type": "email-multiple", - "answer": "boddey@google.com, cmeredith@google.com", - "validation": { - "required": true, - "max": "128" - } + "answer": "Installed in a building" }, { "question": "Is this device going to be managed by Google or a third party?", - "type": "select", - "options": [ - "Google", - "Third Party" - ], - "answer": "Google", - "validation": { - "required": true - } + "answer": "Google" }, { "question": "Will the third-party device administrator be able to grant access to authorized Google personnel upon request?", - "type": "select", - "options": [ - "Yes", - "No", - "N/A" - ], - "default": "N/A", - "answer": "Yes", - "validation": { - "required": true - } + "answer": "Yes" }, { "question": "Are any of the following statements true about your device?", - "description": "This tells us about the data your device will collect", - "type": "select-multiple", "answer": [ 0, 2 - ], - "options": [ - "The device collects any Personal Identifiable Information (PII) or Personal Health Information (PHI)", - "The device collects intellectual property and trade secrets, sensitive business data, critical infrastructure data, identity assets", - "The device stream confidential business data in real-time (seconds)?" ] }, { "question": "Which of the following statements are true about this device?", - "description": "This tells us about the types of data that are transmitted from this device and how the transmission is performed from a technical standpoint.", - "type": "select-multiple", "answer": [ 0, 1, 5 - ], - "options": [ - "PII/PHI, confidential business data, or crown jewel data is transmitted to a destination outside Alphabet's ownership", - "Data transmission occurs across less-trusted networks (e.g. the internet).", - "A failure in data transmission would likely have a substantial negative impact (https://www.rra.rocks/docs/standard_levels#levels-definitions)", - "A confidentiality breach during transmission would have a substantial negative impact", - "The device encrypts data during transmission", - "The device network protocol is well-established and currently used by Google" ] }, { "question": "Does the network protocol assure server-to-client identity verification?", - "type": "select", - "answer": "Yes", - "options": [ - "Yes", - "No", - "I don't know" - ], - "validation": { - "required": true - } + "answer": "Yes" }, { "question": "Click the statements that best describe the characteristics of this device.", - "description": "This tells us about how this device is managed remotely.", - "type": "select-multiple", "answer": [ 0, 1, 2 - ], - "options": [ - "PII/PHI, or confidential business data is accessible from the device without authentication", - "Unrecoverable actions (e.g. disk wipe) can be performed remotely", - "Authentication is required for remote access", - "The management interface is accessible from the public internet", - "Static credentials are used for administration" ] }, { "question": "Are any of the following statements true about this device?", - "description": "This informs us about what other systems and processes this device is a part of.", - "type": "select-multiple", "answer": [ 2, 3 - ], - "options": [ - "The device monitors an environment for active risks to human life.", - "The device is used to convey people, or critical property.", - "The device controls robotics in human-accessible spaces.", - "The device controls physical access systems.", - "The device is involved in processes required by regulations, or compliance. (ex. privacy, security, safety regulations)", - "The device's failure would cause faults in other high-criticality processes." ] } ] diff --git a/testing/api/test_api.py b/testing/api/test_api.py index c59f6eba1..70c1a617f 100644 --- a/testing/api/test_api.py +++ b/testing/api/test_api.py @@ -1499,12 +1499,16 @@ def test_get_profiles(testrun, reset_profiles, add_profile): # pylint: disable= # Send the get request to "/profiles" endpoint r = requests.get(f"{API}/profiles", timeout=5) + # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the response (profiles) response = r.json() + # Check if response is a list assert isinstance(response, list) + # Check if the list is empty assert len(response) == 0 @@ -1515,39 +1519,35 @@ def test_get_profiles(testrun, reset_profiles, add_profile): # pylint: disable= # Send get request to the "/profiles" endpoint r = requests.get(f"{API}/profiles", timeout=5) + # Check if status code is 200 (OK) assert r.status_code == 200 + # Parse the response (profiles) response = r.json() + # Check if response is a list assert isinstance(response, list) + # Check if response contains one profile assert len(response) == 1 # Check that each profile has the expected fields for profile in response: - # Check if "name" key exists in profile - assert "name" in profile - # Check if "status" key exists in profile - assert "status" in profile - # Check if "created" key exists in profile - assert "created" in profile - # Check if "version" key exists in profile - assert "version" in profile - # Check if "questions" key exists in profile - assert "questions" in profile + for field in ["name", "status", "created", "version", "questions", "risk"]: + assert field in profile # Check if "questions" value is a list assert isinstance(profile["questions"], list) - #check that "questions" value has the expected fields + # Check that "questions" value has the expected fields for element in profile["questions"]: # Check if each element is dict assert isinstance(element, dict) + # Check if "question" key is in dict element assert "question" in element - # Check if "type" key is in dict element - assert "type" in element + # Check if "asnswer" key is in dict element assert "answer" in element @@ -1564,8 +1564,10 @@ def test_get_profiles(testrun, reset_profiles, add_profile): # pylint: disable= # Check if status code is 200 (OK) assert r.status_code == 200 + # Check if response is a list assert isinstance(response, list) + # Check if response contains two profiles assert len(response) == 2 From a5b4ffff1754f754819066e173d90f3fd6491fd8 Mon Sep 17 00:00:00 2001 From: Jacob Boddey Date: Tue, 13 Aug 2024 11:53:08 +0100 Subject: [PATCH 29/30] Move validation to session --- framework/python/src/api/api.py | 102 +----------------------- framework/python/src/common/session.py | 105 +++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 101 deletions(-) diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py index 076c0066a..5084555ae 100644 --- a/framework/python/src/api/api.py +++ b/framework/python/src/api/api.py @@ -703,7 +703,7 @@ async def update_profile(self, request: Request, response: Response): return self._generate_msg(False, "Invalid request received") # Validate json profile - if not self._validate_profile_json(req_json): + if not self.get_session().validate_profile_json(req_json): response.status_code = status.HTTP_400_BAD_REQUEST return self._generate_msg(False, "Invalid request received") @@ -774,106 +774,6 @@ async def delete_profile(self, request: Request, response: Response): return self._generate_msg(True, "Successfully deleted that profile") - def _validate_profile_json(self, profile_json): - """Validate properties in profile update requests""" - - # Get the status field - valid = False - if "status" in profile_json and profile_json.get("status") == "Valid": - valid = True - - # Check if "name" exists in profile - if "name" not in profile_json: - LOGGER.error("Missing 'name' in profile") - return False - - # Check if "name" field not empty - elif len(profile_json.get("name").strip()) == 0: - LOGGER.error("Name field left empty") - return False - - # Error handling if "questions" not in request - if "questions" not in profile_json and valid: - LOGGER.error("Missing 'questions' field in profile") - return False - - # Validating the questions section - for question in profile_json.get("questions"): - - # Check if the question field is present - if "question" not in question: - LOGGER.error("The 'question' field is missing") - return False - - # Check if "question" field not empty - elif len(question.get("question").strip()) == 0: - LOGGER.error("A question is missing from 'question' field") - return False - - # Check if question is a recognized question - format_q = self.get_session().get_profile_format_question( - question.get("question")) - - if format_q is None: - LOGGER.error(f"Unrecognized question: {question.get('question')}") - return False - - # Error handling if "answer" is missing - if "answer" not in question and valid: - LOGGER.error("The answer field is missing") - return False - - # If answer is present, check the validation rules - else: - - # Extract the answer out of the profile - answer = question.get("answer") - - # Get the validation rules - field_type = format_q.get("type") - - # Check if type is string or single select, answer should be a string - if ((field_type in ["string", "select"]) - and not isinstance(answer, str)): - LOGGER.error(f"""Answer for question \ -{question.get('question')} is incorrect data type""") - return False - - # Check if type is select, answer must be from list - if field_type == "select": - possible_answers = format_q.get("options") - if answer not in possible_answers: - LOGGER.error(f"""Answer for question \ -{question.get('question')} is not valid""") - return False - - # Validate select multiple field types - if field_type == "select-multiple": - - if not isinstance(answer, list): - LOGGER.error(f"""Answer for question \ -{question.get('question')} is incorrect data type""") - return False - - question_options_len = len(format_q.get("options")) - - # We know it is a list, now check the indexes - for index in answer: - - # Check if the index is an integer - if not isinstance(index, int): - LOGGER.error(f"""Answer for question \ -{question.get('question')} is incorrect data type""") - return False - - # Check if index is 0 or above and less than the num of options - if index < 0 or index >= question_options_len: - LOGGER.error(f"""Invalid index provided as answer for \ -question {question.get('question')}""") - return False - - return True - # Certificates def get_certs(self): LOGGER.debug("Received certs list request") diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index 9cb574eeb..eee203a01 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -449,6 +449,11 @@ def _load_profiles(self): # Parse risk profile json json_data = json.load(f) + # Validate profile JSON + if not self.validate_profile_json(json_data): + LOGGER.error("Profile failed validation") + continue + # Instantiate a new risk profile risk_profile = RiskProfile() @@ -530,6 +535,106 @@ def update_profile(self, profile_json): return risk_profile + def validate_profile_json(self, profile_json): + """Validate properties in profile update requests""" + + # Get the status field + valid = False + if 'status' in profile_json and profile_json.get('status') == 'Valid': + valid = True + + # Check if 'name' exists in profile + if 'name' not in profile_json: + LOGGER.error('Missing "name" in profile') + return False + + # Check if 'name' field not empty + elif len(profile_json.get('name').strip()) == 0: + LOGGER.error('Name field left empty') + return False + + # Error handling if 'questions' not in request + if 'questions' not in profile_json and valid: + LOGGER.error('Missing "questions" field in profile') + return False + + # Validating the questions section + for question in profile_json.get('questions'): + + # Check if the question field is present + if 'question' not in question: + LOGGER.error('The "question" field is missing') + return False + + # Check if 'question' field not empty + elif len(question.get('question').strip()) == 0: + LOGGER.error('A question is missing from "question" field') + return False + + # Check if question is a recognized question + format_q = self.get_profile_format_question( + question.get('question')) + + if format_q is None: + LOGGER.error(f'Unrecognized question: {question.get("question")}') + return False + + # Error handling if 'answer' is missing + if 'answer' not in question and valid: + LOGGER.error('The answer field is missing') + return False + + # If answer is present, check the validation rules + else: + + # Extract the answer out of the profile + answer = question.get('answer') + + # Get the validation rules + field_type = format_q.get('type') + + # Check if type is string or single select, answer should be a string + if ((field_type in ['string', 'select']) + and not isinstance(answer, str)): + LOGGER.error(f'''Answer for question \ +{question.get('question')} is incorrect data type''') + return False + + # Check if type is select, answer must be from list + if field_type == 'select' and valid: + possible_answers = format_q.get('options') + if answer not in possible_answers: + LOGGER.error(f'''Answer for question \ +{question.get('question')} is not valid''') + return False + + # Validate select multiple field types + if field_type == 'select-multiple': + + if not isinstance(answer, list): + LOGGER.error(f'''Answer for question \ +{question.get('question')} is incorrect data type''') + return False + + question_options_len = len(format_q.get('options')) + + # We know it is a list, now check the indexes + for index in answer: + + # Check if the index is an integer + if not isinstance(index, int): + LOGGER.error(f'''Answer for question \ +{question.get('question')} is incorrect data type''') + return False + + # Check if index is 0 or above and less than the num of options + if index < 0 or index >= question_options_len: + LOGGER.error(f'''Invalid index provided as answer for \ +question {question.get('question')}''') + return False + + return True + def delete_profile(self, profile): try: From bbab61127f351e324b74c74833d17deee424ab22 Mon Sep 17 00:00:00 2001 From: J Boddey Date: Tue, 13 Aug 2024 12:57:57 +0100 Subject: [PATCH 30/30] Fix pylint issue Signed-off-by: J Boddey --- framework/python/src/common/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py index eee203a01..77eae8d57 100644 --- a/framework/python/src/common/session.py +++ b/framework/python/src/common/session.py @@ -451,7 +451,7 @@ def _load_profiles(self): # Validate profile JSON if not self.validate_profile_json(json_data): - LOGGER.error("Profile failed validation") + LOGGER.error('Profile failed validation') continue # Instantiate a new risk profile