From e98253e21965fc17fddc02a26845e7b707e98e1d Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:29:52 -0600 Subject: [PATCH 1/2] Revised the JSOn format --- hed/schema/schema_io/schema2json.py | 21 +++- tests/schema/test_json_explicit_attributes.py | 19 ++- tests/schema/test_schema_format_roundtrip.py | 114 ++++++++++++++++++ 3 files changed, 145 insertions(+), 9 deletions(-) diff --git a/hed/schema/schema_io/schema2json.py b/hed/schema/schema_io/schema2json.py index 9e1b374e..a5e6128f 100644 --- a/hed/schema/schema_io/schema2json.py +++ b/hed/schema/schema_io/schema2json.py @@ -399,11 +399,22 @@ def build_attributes_dict(source_dict, include_takes_value): if value: attrs["takesValue"] = value - # List attributes - always include (even if empty) - attrs["suggestedTag"] = get_list_value(HedKey.SuggestedTag, source_dict) - attrs["relatedTag"] = get_list_value(HedKey.RelatedTag, source_dict) - attrs["valueClass"] = get_list_value(HedKey.ValueClass, source_dict) - attrs["unitClass"] = get_list_value(HedKey.UnitClass, source_dict) + # List attributes - only include if non-empty + suggested_tag = get_list_value(HedKey.SuggestedTag, source_dict) + if suggested_tag: + attrs["suggestedTag"] = suggested_tag + + related_tag = get_list_value(HedKey.RelatedTag, source_dict) + if related_tag: + attrs["relatedTag"] = related_tag + + value_class = get_list_value(HedKey.ValueClass, source_dict) + if value_class: + attrs["valueClass"] = value_class + + unit_class = get_list_value(HedKey.UnitClass, source_dict) + if unit_class: + attrs["unitClass"] = unit_class # Single value attributes default_units = source_dict.get(HedKey.DefaultUnits) diff --git a/tests/schema/test_json_explicit_attributes.py b/tests/schema/test_json_explicit_attributes.py index 298acb67..b3795bc8 100644 --- a/tests/schema/test_json_explicit_attributes.py +++ b/tests/schema/test_json_explicit_attributes.py @@ -154,14 +154,25 @@ def test_language_item_inherits_suggested_tag(self): self.assertIn("Sensory-presentation", lang["explicitAttributes"]["suggestedTag"]) def test_empty_lists_omitted(self): - """Test that empty lists are represented as empty arrays, not omitted.""" + """Test that empty lists are omitted from JSON output (not written as empty arrays).""" json_data = self._get_json_output() tags = json_data["tags"] - # Most tags should have relatedTag as empty list + # Tags without relatedTag/valueClass/unitClass should not have these keys at all event = tags["Event"] - self.assertIn("relatedTag", event["attributes"]) - self.assertEqual(event["attributes"]["relatedTag"], []) + # Event has suggestedTag, so it should be present + self.assertIn("suggestedTag", event["attributes"]) + # But if Event doesn't have relatedTag, it should be omitted entirely + if "relatedTag" in event["attributes"]: + # If present, it must be non-empty + self.assertNotEqual(event["attributes"]["relatedTag"], [], + "relatedTag should be omitted entirely, not present as empty list") + + # Check that tags without certain attributes don't have empty lists + item = tags.get("Item", {}) + if "relatedTag" in item.get("attributes", {}): + self.assertNotEqual(item["attributes"]["relatedTag"], [], + "Empty relatedTag should be omitted, not present as []") class TestJSONBackwardsCompatibility(unittest.TestCase): diff --git a/tests/schema/test_schema_format_roundtrip.py b/tests/schema/test_schema_format_roundtrip.py index beb2e3ef..1df229fd 100644 --- a/tests/schema/test_schema_format_roundtrip.py +++ b/tests/schema/test_schema_format_roundtrip.py @@ -253,6 +253,120 @@ def test_library_schema_header_attributes(self): # Version number might be different for unmerged (without library prefix) self.assertEqual(schema.with_standard, schema_unmerged.with_standard) + def test_json_empty_list_attributes_omitted(self): + """Test that empty list attributes (suggestedTag, relatedTag, etc.) are omitted from JSON.""" + import json + + schema = load_schema_version("8.4.0") + json_path = os.path.join(self.temp_dir, "empty_lists.json") + schema.save_as_json(json_path) + + # Read the JSON file and check for empty list attributes + with open(json_path, 'r', encoding='utf-8') as f: + json_data = json.load(f) + + # Check ALL tags for empty lists + tags_with_empty_lists = [] + + for tag_name, tag_data in json_data.get("tags", {}).items(): + # Check attributes dict + attrs = tag_data.get("attributes", {}) + for list_attr in ["suggestedTag", "relatedTag", "valueClass", "unitClass"]: + if list_attr in attrs and attrs[list_attr] == []: + tags_with_empty_lists.append(f"{tag_name}.attributes.{list_attr}") + + # Check explicitAttributes dict + explicit_attrs = tag_data.get("explicitAttributes", {}) + for list_attr in ["suggestedTag", "relatedTag", "valueClass", "unitClass"]: + if list_attr in explicit_attrs and explicit_attrs[list_attr] == []: + tags_with_empty_lists.append(f"{tag_name}.explicitAttributes.{list_attr}") + + self.assertEqual(len(tags_with_empty_lists), 0, + f"Found {len(tags_with_empty_lists)} empty list attributes: {tags_with_empty_lists[:5]}") + + # Verify that tags WITH these attributes have non-empty lists + if "Sensory-event" in json_data.get("tags", {}): + sensory_attrs = json_data["tags"]["Sensory-event"].get("attributes", {}) + if "suggestedTag" in sensory_attrs: + self.assertTrue(len(sensory_attrs["suggestedTag"]) > 0, + "Sensory-event suggestedTag should be non-empty if present") + + def test_extras_sections_roundtrip(self): + """Test that extras sections (Sources, Prefixes, AnnotationPropertyExternal) roundtrip correctly.""" + schema = load_schema_version("8.4.0") + + # Check that original has extras + orig_extras = getattr(schema, 'extras', {}) or {} + self.assertGreater(len(orig_extras), 0, "Schema should have extras sections") + + # Save and reload + json_path = os.path.join(self.temp_dir, "with_extras.json") + schema.save_as_json(json_path) + reloaded = load_schema(json_path) + + # Check reloaded has extras + reloaded_extras = getattr(reloaded, 'extras', {}) or {} + + # Compare each extras section + self.assertEqual(set(orig_extras.keys()), set(reloaded_extras.keys()), + "Extras sections should match") + + for key in orig_extras.keys(): + orig_df = orig_extras[key] + reloaded_df = reloaded_extras[key] + self.assertTrue(orig_df.equals(reloaded_df), + f"Extras section '{key}' should match after roundtrip") + + def test_library_schema_extras_roundtrip(self): + """Test that library schema extras (external annotations, etc.) roundtrip correctly.""" + schema = load_schema_version("score_2.1.0") + + # Check that library schema has extras + orig_extras = getattr(schema, 'extras', {}) or {} + self.assertGreater(len(orig_extras), 0, "Library schema should have extras sections") + + # Check for external annotations specifically + self.assertIn("AnnotationPropertyExternal", orig_extras, + "Library schema should have external annotations") + + # Save and reload + json_path = os.path.join(self.temp_dir, "library_with_extras.json") + schema.save_as_json(json_path, save_merged=False) + reloaded = load_schema(json_path) + + # Check reloaded has all extras + reloaded_extras = getattr(reloaded, 'extras', {}) or {} + self.assertEqual(set(orig_extras.keys()), set(reloaded_extras.keys()), + "Library schema extras sections should match") + + # Verify each extras dataframe matches + for key in orig_extras.keys(): + orig_df = orig_extras[key] + reloaded_df = reloaded_extras[key] + self.assertTrue(orig_df.equals(reloaded_df), + f"Library schema extras '{key}' should match after roundtrip") + + def test_library_schema_score(self): + """Test score library schema roundtrip specifically.""" + schema = load_schema_version("score_2.1.0") + + # Test unmerged format + json_path = os.path.join(self.temp_dir, "score_unmerged.json") + schema.save_as_json(json_path, save_merged=False) + reloaded = load_schema(json_path) + + # Verify library attributes + self.assertEqual(schema.library, reloaded.library) + self.assertEqual(schema.version, reloaded.version) + self.assertEqual(schema.with_standard, reloaded.with_standard) + + # Verify tag counts match + self.assertEqual(len(schema.tags.all_entries), len(reloaded.tags.all_entries)) + + # Verify prologue and epilogue + self.assertEqual(schema.prologue, reloaded.prologue) + self.assertEqual(schema.epilogue, reloaded.epilogue) + if __name__ == "__main__": unittest.main() From 450c112c8c2ca0d5d0339a6c0e7b1ca837de6ff5 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:45:38 -0600 Subject: [PATCH 2/2] Corrected formatting --- tests/schema/test_json_explicit_attributes.py | 10 +-- tests/schema/test_schema_format_roundtrip.py | 79 +++++++++---------- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/tests/schema/test_json_explicit_attributes.py b/tests/schema/test_json_explicit_attributes.py index b3795bc8..68d9f5fb 100644 --- a/tests/schema/test_json_explicit_attributes.py +++ b/tests/schema/test_json_explicit_attributes.py @@ -165,14 +165,14 @@ def test_empty_lists_omitted(self): # But if Event doesn't have relatedTag, it should be omitted entirely if "relatedTag" in event["attributes"]: # If present, it must be non-empty - self.assertNotEqual(event["attributes"]["relatedTag"], [], - "relatedTag should be omitted entirely, not present as empty list") - + self.assertNotEqual( + event["attributes"]["relatedTag"], [], "relatedTag should be omitted entirely, not present as empty list" + ) + # Check that tags without certain attributes don't have empty lists item = tags.get("Item", {}) if "relatedTag" in item.get("attributes", {}): - self.assertNotEqual(item["attributes"]["relatedTag"], [], - "Empty relatedTag should be omitted, not present as []") + self.assertNotEqual(item["attributes"]["relatedTag"], [], "Empty relatedTag should be omitted, not present as []") class TestJSONBackwardsCompatibility(unittest.TestCase): diff --git a/tests/schema/test_schema_format_roundtrip.py b/tests/schema/test_schema_format_roundtrip.py index 1df229fd..386b44ec 100644 --- a/tests/schema/test_schema_format_roundtrip.py +++ b/tests/schema/test_schema_format_roundtrip.py @@ -256,113 +256,112 @@ def test_library_schema_header_attributes(self): def test_json_empty_list_attributes_omitted(self): """Test that empty list attributes (suggestedTag, relatedTag, etc.) are omitted from JSON.""" import json - + schema = load_schema_version("8.4.0") json_path = os.path.join(self.temp_dir, "empty_lists.json") schema.save_as_json(json_path) - + # Read the JSON file and check for empty list attributes - with open(json_path, 'r', encoding='utf-8') as f: + with open(json_path, "r", encoding="utf-8") as f: json_data = json.load(f) - + # Check ALL tags for empty lists tags_with_empty_lists = [] - + for tag_name, tag_data in json_data.get("tags", {}).items(): # Check attributes dict attrs = tag_data.get("attributes", {}) for list_attr in ["suggestedTag", "relatedTag", "valueClass", "unitClass"]: if list_attr in attrs and attrs[list_attr] == []: tags_with_empty_lists.append(f"{tag_name}.attributes.{list_attr}") - + # Check explicitAttributes dict explicit_attrs = tag_data.get("explicitAttributes", {}) for list_attr in ["suggestedTag", "relatedTag", "valueClass", "unitClass"]: if list_attr in explicit_attrs and explicit_attrs[list_attr] == []: tags_with_empty_lists.append(f"{tag_name}.explicitAttributes.{list_attr}") - - self.assertEqual(len(tags_with_empty_lists), 0, - f"Found {len(tags_with_empty_lists)} empty list attributes: {tags_with_empty_lists[:5]}") - + + self.assertEqual( + len(tags_with_empty_lists), + 0, + f"Found {len(tags_with_empty_lists)} empty list attributes: {tags_with_empty_lists[:5]}", + ) + # Verify that tags WITH these attributes have non-empty lists if "Sensory-event" in json_data.get("tags", {}): sensory_attrs = json_data["tags"]["Sensory-event"].get("attributes", {}) if "suggestedTag" in sensory_attrs: - self.assertTrue(len(sensory_attrs["suggestedTag"]) > 0, - "Sensory-event suggestedTag should be non-empty if present") + self.assertTrue( + len(sensory_attrs["suggestedTag"]) > 0, "Sensory-event suggestedTag should be non-empty if present" + ) def test_extras_sections_roundtrip(self): """Test that extras sections (Sources, Prefixes, AnnotationPropertyExternal) roundtrip correctly.""" schema = load_schema_version("8.4.0") - + # Check that original has extras - orig_extras = getattr(schema, 'extras', {}) or {} + orig_extras = getattr(schema, "extras", {}) or {} self.assertGreater(len(orig_extras), 0, "Schema should have extras sections") - + # Save and reload json_path = os.path.join(self.temp_dir, "with_extras.json") schema.save_as_json(json_path) reloaded = load_schema(json_path) - + # Check reloaded has extras - reloaded_extras = getattr(reloaded, 'extras', {}) or {} - + reloaded_extras = getattr(reloaded, "extras", {}) or {} + # Compare each extras section - self.assertEqual(set(orig_extras.keys()), set(reloaded_extras.keys()), - "Extras sections should match") - + self.assertEqual(set(orig_extras.keys()), set(reloaded_extras.keys()), "Extras sections should match") + for key in orig_extras.keys(): orig_df = orig_extras[key] reloaded_df = reloaded_extras[key] - self.assertTrue(orig_df.equals(reloaded_df), - f"Extras section '{key}' should match after roundtrip") + self.assertTrue(orig_df.equals(reloaded_df), f"Extras section '{key}' should match after roundtrip") def test_library_schema_extras_roundtrip(self): """Test that library schema extras (external annotations, etc.) roundtrip correctly.""" schema = load_schema_version("score_2.1.0") - + # Check that library schema has extras - orig_extras = getattr(schema, 'extras', {}) or {} + orig_extras = getattr(schema, "extras", {}) or {} self.assertGreater(len(orig_extras), 0, "Library schema should have extras sections") - + # Check for external annotations specifically - self.assertIn("AnnotationPropertyExternal", orig_extras, - "Library schema should have external annotations") - + self.assertIn("AnnotationPropertyExternal", orig_extras, "Library schema should have external annotations") + # Save and reload json_path = os.path.join(self.temp_dir, "library_with_extras.json") schema.save_as_json(json_path, save_merged=False) reloaded = load_schema(json_path) - + # Check reloaded has all extras - reloaded_extras = getattr(reloaded, 'extras', {}) or {} - self.assertEqual(set(orig_extras.keys()), set(reloaded_extras.keys()), - "Library schema extras sections should match") - + reloaded_extras = getattr(reloaded, "extras", {}) or {} + self.assertEqual(set(orig_extras.keys()), set(reloaded_extras.keys()), "Library schema extras sections should match") + # Verify each extras dataframe matches for key in orig_extras.keys(): orig_df = orig_extras[key] reloaded_df = reloaded_extras[key] - self.assertTrue(orig_df.equals(reloaded_df), - f"Library schema extras '{key}' should match after roundtrip") + self.assertTrue(orig_df.equals(reloaded_df), f"Library schema extras '{key}' should match after roundtrip") def test_library_schema_score(self): """Test score library schema roundtrip specifically.""" schema = load_schema_version("score_2.1.0") - + # Test unmerged format json_path = os.path.join(self.temp_dir, "score_unmerged.json") schema.save_as_json(json_path, save_merged=False) reloaded = load_schema(json_path) - + # Verify library attributes self.assertEqual(schema.library, reloaded.library) self.assertEqual(schema.version, reloaded.version) self.assertEqual(schema.with_standard, reloaded.with_standard) - + # Verify tag counts match self.assertEqual(len(schema.tags.all_entries), len(reloaded.tags.all_entries)) - + # Verify prologue and epilogue self.assertEqual(schema.prologue, reloaded.prologue) self.assertEqual(schema.epilogue, reloaded.epilogue)