diff --git a/common/tools/acs-results-schema.json b/common/tools/acs-results-schema.json index 40d7b3a4..f31ff6f2 100644 --- a/common/tools/acs-results-schema.json +++ b/common/tools/acs-results-schema.json @@ -2,6 +2,10 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ACS Merged Results Schema", "description": "Schema for acs results", + "band-versions": { + "SystemReady Devicetree band": "3.1.2+", + "SystemReady band": "3.1.1+" + }, "type": "object", "required": [ "Suite_Name: acs_info" @@ -39,6 +43,12 @@ }, "Suite_Name: SCMI": { "$ref": "#/definitions/scmi_suite" + }, + "Suite_Name: SBSA": { + "$ref": "#/definitions/bsa_suite" + }, + "Suite_Name: OS Tests": { + "$ref": "#/definitions/os_tests_suite" } }, "patternProperties": { @@ -291,6 +301,56 @@ } ] }, + "os_summary_totals": { + "$comment": "OS tests totals allow both singular and plural waiver counter keys.", + "type": "object", + "properties": { + "total_aborted": { + "$ref": "#/definitions/non_negative_int" + }, + "total_failed": { + "$ref": "#/definitions/non_negative_int" + }, + "total_failed_with_waiver": { + "$ref": "#/definitions/non_negative_int" + }, + "total_failed_with_waivers": { + "$ref": "#/definitions/non_negative_int" + }, + "total_ignored": { + "$ref": "#/definitions/non_negative_int" + }, + "total_passed": { + "$ref": "#/definitions/non_negative_int" + }, + "total_skipped": { + "$ref": "#/definitions/non_negative_int" + }, + "total_warnings": { + "$ref": "#/definitions/non_negative_int" + } + }, + "required": [ + "total_aborted", + "total_failed", + "total_passed", + "total_skipped", + "total_warnings" + ], + "anyOf": [ + { + "required": [ + "total_failed_with_waiver" + ] + }, + { + "required": [ + "total_failed_with_waivers" + ] + } + ], + "additionalProperties": false + }, "subtest_base": { "$comment": "Generic subtest shape used by several suites.", "type": "object", @@ -383,16 +443,6 @@ }, "reason": { "type": "string" - }, - "waiver_reason": { - "oneOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/string_list" - } - ] } }, "required": [ @@ -474,6 +524,9 @@ }, "Waivable": { "type": "string" + }, + "Test_suite_info": { + "$ref": "#/definitions/string_list" } } }, @@ -538,12 +591,19 @@ "type": "object", "required": [ "Test_suite", + "Test_suite_info", "testcases", "test_suite_summary" ], "properties": { "Test_suite": { - "type": "string" + "type": "string", + "not": { + "enum": [ + "Unknown", + "unknown" + ] + } }, "testcases": { "type": "array", @@ -594,7 +654,7 @@ "type": "object", "properties": { "suite_summary": { - "$ref": "#/definitions/bsa_suite_summary" + "$ref": "#/definitions/bsa_suite_summary" }, "test_results": { "type": "array", @@ -621,12 +681,19 @@ "required": [ "Test_suite", "Test_suite_description", + "Test_suite_info", "subtests", "test_suite_summary" ], "properties": { "Test_suite": { - "type": "string" + "type": "string", + "not": { + "enum": [ + "Unknown", + "unknown" + ] + } }, "Test_suite_description": { "type": "string" @@ -685,6 +752,7 @@ "Test_case", "Test_case_description", "Test_suite", + "Test_suite_info", "reason", "subtests", "test_case_summary", @@ -737,6 +805,19 @@ "type": "string" } } + }, + { + "type": "object", + "properties": { + "Test_suite": { + "not": { + "enum": [ + "Unknown", + "unknown" + ] + } + } + } } ], "unevaluatedProperties": false @@ -875,6 +956,7 @@ "required": [ "Test_suite", "Test_suite_description", + "Test_suite_info", "Test_case", "Test_case_description", "subtests", @@ -936,6 +1018,7 @@ "required": [ "Test_suite", "Test_suite_description", + "Test_suite_info", "Test_case", "Test_case_description", "subtests", @@ -961,7 +1044,7 @@ } }, "test_suite_summary": { - "$ref": "#/definitions/summary_totals" + "$ref": "#/definitions/os_summary_totals" } } } @@ -977,7 +1060,7 @@ "type": "object", "properties": { "suite_summary": { - "$ref": "#/definitions/summary_totals" + "$ref": "#/definitions/os_summary_totals" }, "test_results": { "type": "array", @@ -1026,6 +1109,8 @@ "type": "object", "required": [ "Test_suite", + "Test_suite_info", + "reason", "test_suite_summary", "testcases" ], @@ -1033,6 +1118,9 @@ "Test_suite": { "type": "string" }, + "reason": { + "type": "string" + }, "test_suite_summary": { "$ref": "#/definitions/summary_totals" }, @@ -1078,16 +1166,11 @@ "required": [ "ACS version", "Band", - "BBR version", "BBSR version", - "SCMI version", "BSA version", - "Device Tree Version", - "EBBR version", "Firmware Version", "Flashing instructions", "FW source code", - "PFDI version", "product website", "SoC Family", "SRS version", @@ -1111,6 +1194,9 @@ "BBSR version": { "type": "string" }, + "BMC Firmware Version": { + "type": "string" + }, "SCMI version": { "type": "string" }, @@ -1159,10 +1245,62 @@ "UEFI Version": { "type": "string" }, + "SBBR version": { + "type": "string" + }, + "SBMR version": { + "type": "string" + }, + "SBSA version": { + "type": "string" + }, "Vendor": { "type": "string" } }, + "allOf": [ + { + "if": { + "properties": { + "Band": { + "const": "SystemReady Devicetree band" + } + }, + "required": [ + "Band" + ] + }, + "then": { + "required": [ + "BBR version", + "SCMI version", + "Device Tree Version", + "EBBR version", + "PFDI version" + ] + } + }, + { + "if": { + "properties": { + "Band": { + "const": "SystemReady band" + } + }, + "required": [ + "Band" + ] + }, + "then": { + "required": [ + "BMC Firmware Version", + "SBBR version", + "SBMR version", + "SBSA version" + ] + } + } + ], "additionalProperties": false }, "acs_results_summary": { @@ -1213,9 +1351,6 @@ "Suite_Name: Mandatory : ETHTOOL_TEST_compliance": { "type": "string" }, - "Suite_Name: Recommended : RUNTIME_DEV_MAP_compliance": { - "type": "string" - }, "Suite_Name: Mandatory : FWTS_compliance": { "type": "string" }, @@ -1249,10 +1384,107 @@ "Suite_Name: Recommended : PSCI_compliance": { "type": "string" }, + "Suite_Name: Recommended : RUNTIME_DEV_MAP_compliance": { + "type": "string" + }, "Suite_Name: Recommended : SMBIOS_compliance": { "type": "string" + }, + "Suite_Name: Mandatory : BSA_compliance": { + "type": "string" + }, + "Suite_Name: Mandatory : OS_TEST_compliance": { + "type": "string" + }, + "Suite_Name: Recommended : SBMR-IB_compliance": { + "type": "string" + }, + "Suite_Name: Recommended : SBMR-OOB_compliance": { + "type": "string" + }, + "Suite_Name: Recommended : SBSA_compliance": { + "type": "string" + }, + "Suite_Name: Mandatory : SBSA_compliance": { + "type": "string" } }, + "allOf": [ + { + "if": { + "properties": { + "Band": { + "const": "SystemReady Devicetree band" + } + }, + "required": [ + "Band" + ] + }, + "then": { + "required": [ + "Suite_Name: Mandatory : Capsule Update_compliance", + "Suite_Name: Mandatory : DT_VALIDATE_compliance", + "Suite_Name: Mandatory : ETHTOOL_TEST_compliance", + "Suite_Name: Mandatory : FWTS_compliance", + "Suite_Name: Mandatory : READ_WRITE_CHECK_BLK_DEVICES_compliance", + "Suite_Name: Mandatory : SCT_compliance", + "Suite_Name: Mandatory : OS_ethtool_test_linux-Fedora-40_compliance", + "Suite_Name: Mandatory : OS_ethtool_test_linux-Tumbleweed_compliance", + "Suite_Name: Mandatory : OS_ethtool_test_linux-ubuntu-24_compliance", + "Suite_Name: Recommended : BSA_compliance", + "Suite_Name: Recommended : DT_KSELFTEST_compliance", + "Suite_Name: Recommended : NETWORK_BOOT_compliance", + "Suite_Name: Recommended : POST_SCRIPT_compliance", + "Suite_Name: Recommended : PSCI_compliance", + "Suite_Name: Recommended : RUNTIME_DEV_MAP_compliance", + "Suite_Name: Recommended : SMBIOS_compliance", + "Suite_Name: Extension : SCMI_compliance", + "Suite_Name: Conditional-Mandatory : PFDI_compliance", + "Suite_Name: Extension : BBSR-FWTS_compliance", + "Suite_Name: Extension : BBSR-SCT_compliance", + "Suite_Name: Extension : BBSR-TPM_compliance" + ] + } + }, + { + "if": { + "properties": { + "Band": { + "const": "SystemReady band" + } + }, + "required": [ + "Band" + ] + }, + "then": { + "required": [ + "Suite_Name: Mandatory : BSA_compliance", + "Suite_Name: Mandatory : FWTS_compliance", + "Suite_Name: Mandatory : OS_TEST_compliance", + "Suite_Name: Mandatory : SCT_compliance", + "Suite_Name: Recommended : SBMR-IB_compliance", + "Suite_Name: Recommended : SBMR-OOB_compliance", + "Suite_Name: Extension : BBSR-FWTS_compliance", + "Suite_Name: Extension : BBSR-SCT_compliance", + "Suite_Name: Extension : BBSR-TPM_compliance" + ], + "anyOf": [ + { + "required": [ + "Suite_Name: Recommended : SBSA_compliance" + ] + }, + { + "required": [ + "Suite_Name: Mandatory : SBSA_compliance" + ] + } + ] + } + } + ], "additionalProperties": false }, "acs_info": { @@ -1277,7 +1509,16 @@ "$ref": "#/definitions/test_result_base" }, { + "required": [ + "subtests" + ], "properties": { + "subtests": { + "type": "array", + "items": { + "$ref": "#/definitions/subtest_base" + } + }, "test_suite_summary": { "$ref": "#/definitions/summary_totals" } diff --git a/common/tools/validate.sh b/common/tools/validate.sh index 329b5971..4596eb42 100755 --- a/common/tools/validate.sh +++ b/common/tools/validate.sh @@ -79,8 +79,10 @@ echo -e "${BLUE}=====================================${NC}\n" # Run validation with concise, readable errors (path + message) output=$(python3 - "$JSON_FILE" "$SCHEMA_FILE" <<'PY' import json +import re import sys -from jsonschema import Draft7Validator +from jsonschema import Draft202012Validator +from jsonschema.exceptions import best_match # ANSI colors RED = "\033[0;31m" @@ -109,6 +111,96 @@ def suite_from_path(path): return first return "" +def collect_key_issues(error): + missing = set() + unexpected = set() + + def handle(err): + if err.validator == "required" and isinstance(err.instance, dict): + required = err.validator_value if isinstance(err.validator_value, list) else [] + for key in required: + if key not in err.instance: + missing.add(key) + elif err.validator == "additionalProperties" and isinstance(err.message, str): + for key in re.findall(r"'([^']+)'", err.message): + unexpected.add(key) + + if error.context: + for sub in error.context: + handle(sub) + else: + handle(error) + + return missing, unexpected + +def best_suberror(error): + if error.validator not in ("anyOf", "oneOf") or not error.context: + return None + # Prefer a concrete schema failure over the fallback additionalProperties noise. + non_additional = [e for e in error.context if e.validator != "additionalProperties"] + candidate = best_match(non_additional) if non_additional else None + # If best match is a type mismatch but additionalProperties exists, surface the key error. + if candidate is not None and candidate.validator == "type": + for e in error.context: + if e.validator == "additionalProperties": + return e + if candidate is not None: + return candidate + return best_match(error.context) + + +def error_tag(error, missing, unexpected): + if missing: + return "MISSING_KEY" + if unexpected: + return "UNEXPECTED_KEY" + if error.validator == "not": + return "DISALLOWED_VALUE" + if error.validator == "type": + return "TYPE_MISMATCH" + if error.validator == "enum": + return "ENUM" + if error.validator: + return str(error.validator).upper() + return "VALIDATION" + +def shorten_message(error, message): + # Reduce noisy object/array dumps in messages like "{...} is not of type ..." + if isinstance(error.instance, dict) and message.startswith("{") and " is " in message: + return "object" + message[message.find(" is "):] + if isinstance(error.instance, list) and message.startswith("[") and " is " in message: + return "array" + message[message.find(" is "):] + # Normalize "not" errors when a value is explicitly disallowed via enum. + if error.validator == "not": + enum_vals = [] + if isinstance(error.schema, dict): + not_schema = error.schema.get("not") + if isinstance(not_schema, dict): + enum_vals = not_schema.get("enum") or [] + if enum_vals: + return f"value '{error.instance}' is not allowed" + return message + +def subtest_result_unexpected_keys(instance, schema): + # Generic: detect unexpected keys inside sub_test_result objects. + try: + allowed = set(schema["definitions"]["sub_test_result_object"]["properties"].keys()) + except Exception: + return set() + if not isinstance(instance, dict): + return set() + subtests = instance.get("subtests") + if not isinstance(subtests, list): + return set() + unexpected = set() + for sub in subtests: + if not isinstance(sub, dict): + continue + sr = sub.get("sub_test_result") + if isinstance(sr, dict): + unexpected.update(set(sr.keys()) - allowed) + return unexpected + json_file = sys.argv[1] schema_file = sys.argv[2] @@ -117,15 +209,61 @@ with open(schema_file, "r", encoding="utf-8") as sf: with open(json_file, "r", encoding="utf-8") as jf: instance = json.load(jf) -v = Draft7Validator(schema) +v = Draft202012Validator(schema) errors = sorted(v.iter_errors(instance), key=lambda e: list(e.path)) if not errors: sys.exit(0) +def is_prefix_path(prefix, full): + if len(prefix) > len(full): + return False + return all(p == f for p, f in zip(prefix, full)) + +# Suppress cascading unevaluatedProperties when a more specific child error exists. +by_suite = {} +for e in errors: + s = suite_from_path(list(e.path)) + by_suite.setdefault(s, []).append(e) + +filtered = [] +for suite, errs in by_suite.items(): + other_paths = [list(e.path) for e in errs if e.validator != "unevaluatedProperties"] + for e in errs: + if e.validator == "unevaluatedProperties" and other_paths: + ep = list(e.path) + if any(is_prefix_path(ep, op) for op in other_paths): + continue + filtered.append(e) + +errors = sorted(filtered, key=lambda e: list(e.path)) + suite_errors = {} for err in errors: suite = suite_from_path(list(err.path)) - key = (suite, err.message) + sub = best_suberror(err) + base_err = sub if sub is not None else err + msg = shorten_message(base_err, base_err.message) + missing, unexpected = collect_key_issues(base_err) + + # If only a cascading unevaluatedProperties error, surface unexpected keys + # from nested sub_test_result objects (generic across suites). + if base_err.validator == "unevaluatedProperties" and not missing and not unexpected: + sub_unexpected = subtest_result_unexpected_keys(err.instance, schema) + if sub_unexpected: + unexpected = sub_unexpected + msg = "Additional properties are not allowed (" + ", ".join( + f"'{k}' was unexpected" for k in sorted(sub_unexpected) + ) + ")" + details = [] + if missing: + details.append(f"{RED}missing{NC}: " + ", ".join(sorted(missing))) + if unexpected: + details.append(f"{BLUE}unexpected{NC}: " + ", ".join(sorted(unexpected))) + if details: + msg = f"{msg} ({'; '.join(details)})" + tag = error_tag(base_err, missing, unexpected) + msg = f"{YELLOW}{tag}{NC}: {msg}" + key = (suite, msg) suite_errors.setdefault(key, []).append(err) suites = [k for k in instance.keys() if isinstance(k, str) and k.startswith("Suite_Name:")] @@ -147,6 +285,7 @@ for suite in suites: print(f"{YELLOW} *at={p}{NC}") if len(paths) > 5: print(f"{YELLOW} *... and {len(paths) - 5} more{NC}") + print() if not any_err: print(f"{GREEN}*suite={suite} no errors{NC}")