From ebeaaac29272c7b46ef1a89d5cabc3422cbd775f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Fri, 3 May 2019 17:21:39 +0200 Subject: [PATCH 1/7] Add test wrapper around SCAPVal tool Runs SCAPVal tool on all built datastreams and then parses the results. This wrapper will run in a special job in our Jenkins, which will help us to preserve SCAP 1.3 conformance. Example usage: python run_scapval.py --scap-version 1.3 \ --scapval-path /opt/scapval/scapval-1.3.2.jar \ --build-dir content/build --- tests/run_scapval.py | 113 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 tests/run_scapval.py diff --git a/tests/run_scapval.py b/tests/run_scapval.py new file mode 100755 index 000000000000..faeef7008c71 --- /dev/null +++ b/tests/run_scapval.py @@ -0,0 +1,113 @@ +#!/usr/bin/python + +from __future__ import print_function + +import argparse +import subprocess +import os +import xml.etree.ElementTree as ET +import sys + +NS = "http://csrc.nist.gov/ns/decima/results/1.0" +oval_unix_ns = "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix" +xccdf_ns = "http://checklists.nist.gov/xccdf/1.2" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Runs SCAP Validation of our data streams using SCAP" + "Validation Tool (SCAPVal)") + parser.add_argument( + "--scap-version", + help="SCAP Version (Only 1.2 and 1.3 supported)", + choices=["1.2", "1.3"], required=True) + parser.add_argument( + "--scapval-path", + help="Full path to the SCAPVal JAR archive", required=True) + parser.add_argument( + "--build-dir", + help="Full path to the ComplianceAsCode build directory", + required=True) + return parser.parse_args() + + +def process_results(result_path): + ret_val = True + tree = ET.parse(result_path) + root = tree.getroot() + results = root.find("./{%s}results" % NS) + for base_req in results.findall("./{%s}base-requirement" % NS): + id_ = base_req.get("id") + status = base_req.find("./{%s}status" % NS).text + if status == "FAIL": + print(" %s: %s" % (id_, status)) + ret_val = False + return ret_val + + +def workaround_datastream(datastream_path): + tree = ET.parse(datastream_path) + root = tree.getroot() + # group_id and user_id cannot be zero + # tracked at https://github.com/OVAL-Community/OVAL/issues/23 + for group_id_element in root.findall(".//{%s}group_id" % oval_unix_ns): + if group_id_element.text is not None: + group_id_element.text = "-1" + for user_id_element in root.findall(".//{%s}user_id" % oval_unix_ns): + if user_id_element.text is not None: + user_id_element.text = "-1" + # OCIL checks for security_patches_up_to_date is causing fail + # of SRC-377, when requirement is about OVAL checks. + rule_id = "xccdf_org.ssgproject.content_rule_security_patches_up_to_date" + for rule in root.findall(".//{%s}Rule[@id=\"%s\"]" % (xccdf_ns, rule_id)): + for check in rule.findall("{%s}check" % xccdf_ns): + system = check.get("system") + if system == "http://scap.nist.gov/schema/ocil/2": + rule.remove(check) + output_path = datastream_path + ".workaround.xml" + tree.write(output_path) + return output_path + + +def test_datastream(datastream_path, scapval_path, scap_version): + result_path = datastream_path + ".result.xml" + if scap_version == "1.3": + datastream_path = workaround_datastream(datastream_path) + scapval_command = [ + "java", + "-Xmx1024m", + "-jar", scapval_path, + "-scapversion", scap_version, + "-file", datastream_path, + "-valresultfile", result_path + ] + subprocess.call(scapval_command) + return process_results(result_path) + + +def main(): + overall_result = True + args = parse_args() + if args.scap_version == "1.2": + ds_suffix = "-ds.xml" + elif args.scap_version == "1.3": + ds_suffix = "-ds-1.3.xml" + for filename in os.listdir(args.build_dir): + if filename.endswith(ds_suffix): + print("Testing %s ..." % filename) + datastream_path = os.path.join(args.build_dir, filename) + datastream_result = test_datastream( + datastream_path, args.scapval_path, args.scap_version) + if datastream_result: + print("%s: PASS" % filename) + else: + print("%s: FAIL" % filename) + overall_result = False + if overall_result: + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == "__main__": + main() From 5e9b9ac3943bf6c5f8ca99ac5981ec7703943ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Mon, 6 May 2019 10:58:08 +0200 Subject: [PATCH 2/7] Rename namespace variable to be more specific --- tests/run_scapval.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/run_scapval.py b/tests/run_scapval.py index faeef7008c71..2240accf1d64 100755 --- a/tests/run_scapval.py +++ b/tests/run_scapval.py @@ -8,7 +8,7 @@ import xml.etree.ElementTree as ET import sys -NS = "http://csrc.nist.gov/ns/decima/results/1.0" +SCAPVAL_RESULTS_NS = "http://csrc.nist.gov/ns/decima/results/1.0" oval_unix_ns = "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix" xccdf_ns = "http://checklists.nist.gov/xccdf/1.2" @@ -35,10 +35,11 @@ def process_results(result_path): ret_val = True tree = ET.parse(result_path) root = tree.getroot() - results = root.find("./{%s}results" % NS) - for base_req in results.findall("./{%s}base-requirement" % NS): + results = root.find("./{%s}results" % SCAPVAL_RESULTS_NS) + for base_req in results.findall( + "./{%s}base-requirement" % SCAPVAL_RESULTS_NS): id_ = base_req.get("id") - status = base_req.find("./{%s}status" % NS).text + status = base_req.find("./{%s}status" % SCAPVAL_RESULTS_NS).text if status == "FAIL": print(" %s: %s" % (id_, status)) ret_val = False From 02f98421644a6184a6b33edd9d41ba083799d04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Mon, 6 May 2019 10:58:33 +0200 Subject: [PATCH 3/7] Do not pollute the output with SCAPVal output --- tests/run_scapval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_scapval.py b/tests/run_scapval.py index 2240accf1d64..74a8392dd543 100755 --- a/tests/run_scapval.py +++ b/tests/run_scapval.py @@ -82,7 +82,7 @@ def test_datastream(datastream_path, scapval_path, scap_version): "-file", datastream_path, "-valresultfile", result_path ] - subprocess.call(scapval_command) + subprocess.check_output(scapval_command, stderr=subprocess.STDOUT) return process_results(result_path) From c996380b5146dd005dcbc17a4d96889035b7a106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Mon, 6 May 2019 11:25:19 +0200 Subject: [PATCH 4/7] Rename a namespace variable consistently --- tests/run_scapval.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/run_scapval.py b/tests/run_scapval.py index 74a8392dd543..fd5b0402f9fb 100755 --- a/tests/run_scapval.py +++ b/tests/run_scapval.py @@ -8,7 +8,7 @@ import xml.etree.ElementTree as ET import sys -SCAPVAL_RESULTS_NS = "http://csrc.nist.gov/ns/decima/results/1.0" +scapval_results_ns = "http://csrc.nist.gov/ns/decima/results/1.0" oval_unix_ns = "http://oval.mitre.org/XMLSchema/oval-definitions-5#unix" xccdf_ns = "http://checklists.nist.gov/xccdf/1.2" @@ -35,11 +35,11 @@ def process_results(result_path): ret_val = True tree = ET.parse(result_path) root = tree.getroot() - results = root.find("./{%s}results" % SCAPVAL_RESULTS_NS) + results = root.find("./{%s}results" % scapval_results_ns) for base_req in results.findall( - "./{%s}base-requirement" % SCAPVAL_RESULTS_NS): + "./{%s}base-requirement" % scapval_results_ns): id_ = base_req.get("id") - status = base_req.find("./{%s}status" % SCAPVAL_RESULTS_NS).text + status = base_req.find("./{%s}status" % scapval_results_ns).text if status == "FAIL": print(" %s: %s" % (id_, status)) ret_val = False From e9f1124a30a958e2122824bbc037fded121def38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Mon, 6 May 2019 11:25:40 +0200 Subject: [PATCH 5/7] Create a SCAPVal HTML report --- tests/run_scapval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/run_scapval.py b/tests/run_scapval.py index fd5b0402f9fb..7fd048fbde4e 100755 --- a/tests/run_scapval.py +++ b/tests/run_scapval.py @@ -72,6 +72,7 @@ def workaround_datastream(datastream_path): def test_datastream(datastream_path, scapval_path, scap_version): result_path = datastream_path + ".result.xml" + report_path = datastream_path + ".report.xml" if scap_version == "1.3": datastream_path = workaround_datastream(datastream_path) scapval_command = [ @@ -80,7 +81,8 @@ def test_datastream(datastream_path, scapval_path, scap_version): "-jar", scapval_path, "-scapversion", scap_version, "-file", datastream_path, - "-valresultfile", result_path + "-valresultfile", result_path, + "-valreportfile", report_path ] subprocess.check_output(scapval_command, stderr=subprocess.STDOUT) return process_results(result_path) From 587b5bc21dbcf535fcf713c319a1af3c28f3e79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Mon, 6 May 2019 11:31:04 +0200 Subject: [PATCH 6/7] Fix extension of HTML report --- tests/run_scapval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_scapval.py b/tests/run_scapval.py index 7fd048fbde4e..58685c91f97f 100755 --- a/tests/run_scapval.py +++ b/tests/run_scapval.py @@ -72,7 +72,7 @@ def workaround_datastream(datastream_path): def test_datastream(datastream_path, scapval_path, scap_version): result_path = datastream_path + ".result.xml" - report_path = datastream_path + ".report.xml" + report_path = datastream_path + ".report.html" if scap_version == "1.3": datastream_path = workaround_datastream(datastream_path) scapval_command = [ From 144c9eb924f0286fe6f7aee9a448107649a66cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Thu, 9 May 2019 11:37:25 +0200 Subject: [PATCH 7/7] Catch CalledProcessError exception to print the outuput It would be useful to see the SCAPVAl stderr in the test output to determine why SCAPVal failed. --- tests/run_scapval.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/run_scapval.py b/tests/run_scapval.py index 58685c91f97f..4ae223cd8266 100755 --- a/tests/run_scapval.py +++ b/tests/run_scapval.py @@ -84,7 +84,12 @@ def test_datastream(datastream_path, scapval_path, scap_version): "-valresultfile", result_path, "-valreportfile", report_path ] - subprocess.check_output(scapval_command, stderr=subprocess.STDOUT) + try: + subprocess.check_output(scapval_command, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + sys.stderr.write("Command '{0}' returned {1}:\n{2}\n".format( + " ".join(e.cmd), e.returncode, e.output.decode("utf-8"))) + sys.exit(1) return process_results(result_path)