From 042c1695a3db3e4c2f8118722ae23b74b4123c83 Mon Sep 17 00:00:00 2001 From: IanCa Date: Fri, 3 Mar 2023 16:37:25 -0600 Subject: [PATCH 1/3] Update spec tests further --- .github/workflows/spec_tests.yaml | 16 +++-- .gitmodules | 4 ++ spec_tests/test_errors.py | 116 ++++++++++++++++++++++++------ 3 files changed, 109 insertions(+), 27 deletions(-) create mode 100644 .gitmodules diff --git a/.github/workflows/spec_tests.yaml b/.github/workflows/spec_tests.yaml index 7c56db1c0..e41edd61f 100644 --- a/.github/workflows/spec_tests.yaml +++ b/.github/workflows/spec_tests.yaml @@ -18,13 +18,8 @@ jobs: steps: - name: Checkout hed-python uses: actions/checkout@v3 - - - name: Checkout spec - uses: actions/checkout@v3 with: - repository: hed-standard/hed-specification - ref: develop - path: hed-specification + submodules: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -38,5 +33,12 @@ jobs: - name: Test with unittest run: | - python -m unittest spec_tests/* + python -m unittest spec_tests/* > test_results.txt + continue-on-error: true + + - name: Upload spec test results + uses: actions/upload-artifact@v3 + with: + name: spec-test-results + path: test_results.txt diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..b4a35641d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "spec_tests/hed-specification"] + path = spec_tests/hed-specification + url = https://github.com/hed-standard/hed-specification/ + branch = develop diff --git a/spec_tests/test_errors.py b/spec_tests/test_errors.py index 4027d985f..f43bc9c86 100644 --- a/spec_tests/test_errors.py +++ b/spec_tests/test_errors.py @@ -5,16 +5,21 @@ from hed.models.hed_ops import apply_ops from hed import load_schema_version from hed import HedValidator +from hed import Sidecar +import io +import json class MyTestCase(unittest.TestCase): @classmethod def setUpClass(cls): test_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), - '../hed-specification/docs/source/_static/data/error_tests')) + 'hed-specification/docs/source/_static/data/error_tests')) cls.test_files = [os.path.join(test_dir, f) for f in os.listdir(test_dir) if os.path.isfile(os.path.join(test_dir, f))] cls.fail_count = 0 + cls.default_sidecar = Sidecar(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_sidecar.json'))) + def run_single_test(self, test_file): with open(test_file, "r") as fp: @@ -24,6 +29,7 @@ def run_single_test(self, test_file): if error_code == "VERSION_DEPRECATED": print("Skipping VERSION_DEPRECATED test") continue + name = info.get('name', '') description = info['description'] schema = info['schema'] if schema: @@ -37,32 +43,102 @@ def run_single_test(self, test_file): validator = HedValidator(schema) def_mapper = DefMapper(def_dict) onset_mapper = OnsetMapper(def_mapper) - for section_name in info["tests"]: + for section_name, section in info["tests"].items(): if section_name == "string_tests": - for result, tests in info["tests"]["string_tests"].items(): - for test in tests: - modified_test, issues = apply_ops(test, [validator, def_mapper, onset_mapper], check_for_warnings=True, expand_defs=True) - if modified_test and modified_test != test: - _, def_expand_issues = apply_ops(modified_test, validator, check_for_warnings=True) - issues += def_expand_issues - if result == "fails": - if not issues: - print(f"{error_code}: {description}") - print(f"Passed this test(that should fail): {test}") - print(issues) - self.fail_count += 1 - else: - if issues: - print(f"{error_code}: {description}") - print(f"Failed this test: {test}") - print(issues) - self.fail_count += 1 + self._run_single_string_test(section, validator, def_mapper, + onset_mapper, error_code, description, name) + elif section_name == "sidecar_tests": + self._run_single_sidecar_test(section, validator, def_mapper, onset_mapper, error_code, description, + name) + elif section_name == "event_tests": + self._run_single_events_test(section, validator, def_mapper, onset_mapper, error_code, description, + name) + + def _run_single_string_test(self, info, validator, def_mapper, onset_mapper, error_code, description, + name): + for result, tests in info.items(): + for test in tests: + modified_test, issues = apply_ops(test, [validator, def_mapper, onset_mapper], check_for_warnings=True, + expand_defs=True) + if modified_test and modified_test != test: + _, def_expand_issues = apply_ops(modified_test, validator, check_for_warnings=True) + issues += def_expand_issues + if result == "fails": + if not issues: + print(f"{error_code}: {description}") + print(f"Passed this test(that should fail) '{name}': {test}") + print(issues) + self.fail_count += 1 + else: + if issues: + print(f"{error_code}: {description}") + print(f"Failed this test {name}: {test}") + print(issues) + + self.fail_count += 1 + + def _run_single_sidecar_test(self, info, validator, def_mapper, onset_mapper, error_code, description, + name): + for result, tests in info.items(): + + for test in tests: + # Well this is a disaster + buffer = io.BytesIO(json.dumps(test).encode("utf-8")) + sidecar = Sidecar(buffer) + issues = sidecar.validate_entries([validator, def_mapper, onset_mapper], check_for_warnings=True) + if result == "fails": + if not issues: + print(f"{error_code}: {description}") + print(f"Passed this test(that should fail) '{name}': {test}") + print(issues) + self.fail_count += 1 + else: + if issues: + print(f"{error_code}: {description}") + print(f"Failed this test {name}: {test}") + print(issues) + + self.fail_count += 1 + + def _run_single_events_test(self, info, validator, def_mapper, onset_mapper, error_code, description, + name): + from hed import TabularInput + for result, tests in info.items(): + + for test in tests: + string = "" + for row in test: + if not isinstance(row, list): + print(f"Improper grouping in test: {error_code}:{name}") + print(f"This is probably a missing set of square brackets.") + break + string += "\t".join(str(x) for x in row) + "\n" + + if not string: + print(F"Invalid blank events found in test: {error_code}:{name}") + continue + file_obj = io.BytesIO(string.encode("utf-8")) + file = TabularInput(file_obj, sidecar=self.default_sidecar) + issues = file.validate_file([validator, def_mapper, onset_mapper], check_for_warnings=True) + if result == "fails": + if not issues: + print(f"{error_code}: {description}") + print(f"Passed this test(that should fail) '{name}': {test}") + print(issues) + self.fail_count += 1 + else: + if issues: + print(f"{error_code}: {description}") + print(f"Failed this test {name}: {test}") + print(issues) + self.fail_count += 1 def test_summary(self): for test_file in self.test_files: self.run_single_test(test_file) + print(f"{self.fail_count} tests got an unexpected result") self.assertEqual(self.fail_count, 0) if __name__ == '__main__': From c934d93a1d6b393b1e672930b037bda93e718efe Mon Sep 17 00:00:00 2001 From: IanCa Date: Fri, 3 Mar 2023 16:45:15 -0600 Subject: [PATCH 2/3] Add specification subfolder --- spec_tests/hed-specification | 1 + 1 file changed, 1 insertion(+) create mode 160000 spec_tests/hed-specification diff --git a/spec_tests/hed-specification b/spec_tests/hed-specification new file mode 160000 index 000000000..9e7d37eb4 --- /dev/null +++ b/spec_tests/hed-specification @@ -0,0 +1 @@ +Subproject commit 9e7d37eb497a44d0bf6068633077d94788e79792 From 22066bf5d5388c8cfef5bbe3cbebfa9eaf9f72fc Mon Sep 17 00:00:00 2001 From: IanCa Date: Fri, 3 Mar 2023 16:46:50 -0600 Subject: [PATCH 3/3] Add missing file --- spec_tests/test_sidecar.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 spec_tests/test_sidecar.json diff --git a/spec_tests/test_sidecar.json b/spec_tests/test_sidecar.json new file mode 100644 index 000000000..b84e0bdcd --- /dev/null +++ b/spec_tests/test_sidecar.json @@ -0,0 +1,10 @@ +{ + "onset": { + "Description": "Position of event marker in seconds relative to the start.", + "Units": "s" + }, + "duration": { + "Description": "Duration of the event in seconds.", + "Units": "s" + } +} \ No newline at end of file