From a13418a18d1e9473500a73b19a821da9a30aab1c Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Fri, 8 Jul 2022 00:02:26 -0400 Subject: [PATCH 01/28] Upstream validation features up to and including: e956afc69f7901a5848d2d91187d3036a6d7eb4b --- dandi/bids_validator_xs.py | 161 ++++++++++++++++---------- dandi/tests/fixtures.py | 2 +- dandi/tests/test_bids_validator_xs.py | 76 +++++------- 3 files changed, 132 insertions(+), 107 deletions(-) diff --git a/dandi/bids_validator_xs.py b/dandi/bids_validator_xs.py index 3de0bde3d..3e0cd95cf 100644 --- a/dandi/bids_validator_xs.py +++ b/dandi/bids_validator_xs.py @@ -1,3 +1,7 @@ +""" +THIS FILE IS BUNDLED FROM THE bids-specification PACKAGE. +Schema loading- and processing-related functions. +""" from copy import deepcopy import datetime from functools import lru_cache @@ -5,8 +9,6 @@ import os import re -import appdirs - from . import utils from .support.bids import schema @@ -19,7 +21,10 @@ DIR_ENTITIES = ["subject", "session"] -def _get_paths(bids_paths): +def _get_paths( + bids_paths, + pseudofile_suffixes=[], +): """ Get all paths from a list of directories, excluding hidden subdirectories from distribution. @@ -28,6 +33,9 @@ def _get_paths(bids_paths): bids_paths : list or str Directories from which to get paths, may also contain file paths, which will remain unchanged. + pseudofile_suffixes : list of str + Directory suffixes prompting the validation of the directory name and limiting further + directory walk. Notes ----- @@ -50,9 +58,6 @@ def _get_paths(bids_paths): ".bidsignore", "dandiset.yaml", ] - # Inelegant hard-coded solution. - # Could be replaced by a maximum depth limit if BIDS root auto-detection is implemented. - treat_as_file_suffix = [".ngff"] path_list = [] for bids_path in bids_paths: @@ -60,13 +65,12 @@ def _get_paths(bids_paths): if os.path.isfile(bids_path): path_list.append(bids_path) continue - for root, dirs, file_names in os.walk(bids_path, topdown=False): - if any(root.endswith(i) for i in treat_as_file_suffix): - continue - if any(f"{i}/" in root for i in treat_as_file_suffix): - continue - if any(f"{i}\\" in root for i in treat_as_file_suffix): - continue + for root, dirs, file_names in os.walk(bids_path, topdown=True): + if any(root.endswith(i) for i in pseudofile_suffixes): + # Add the directory name to the validation paths list. + path_list.append(f"{root}/") + # Do not index the contents of the directory. + dirs[:] = [] # will break if BIDS ever puts meaningful data under `/.{dandi,datalad,git}*/` if any(exclude_subdir in root for exclude_subdir in exclude_subdirs): continue @@ -160,20 +164,20 @@ def _add_extensions(regex_string, variant): def _add_subdirs( - regex_string, variant, datatype, entity_definitions, modality_datatypes + regex_string, variant, datatype, entity_definitions, formats, modality_datatypes ): """Add appropriate subdirectories as required by entities present.""" - label = "([a-zA-Z0-9]*?)" - regex_dirs = "/" for dir_entity in DIR_ENTITIES: if dir_entity in variant["entities"].keys(): - shorthand = entity_definitions[dir_entity]["entity"] + format_selection = formats[entity_definitions[dir_entity]["format"]] + variable_field = f"({format_selection['pattern']})" + shorthand = entity_definitions[dir_entity]["name"] if variant["entities"][dir_entity] == "required": - regex_subdir = f"{shorthand}-(?P<{dir_entity}>{label})/" + regex_subdir = f"{shorthand}-(?P<{dir_entity}>{variable_field})/" else: - regex_subdir = f"(|{shorthand}-(?P<{dir_entity}>{label})/)" + regex_subdir = f"(|{shorthand}-(?P<{dir_entity}>{variable_field})/)" regex_dirs = f"{regex_dirs}{regex_subdir}" if datatype in modality_datatypes: regex_dirs = f"{regex_dirs}{datatype}/" @@ -204,7 +208,7 @@ def load_top_level( Parameters ---------- my_schema : dict - A nested dictionary, as returned by `dandi.support.bids.schema.load_schema()`. + A nested dictionary, as returned by `schemacode.schema.load_schema()`. Returns ------- @@ -241,17 +245,11 @@ def load_entities( Parameters ---------- my_schema : dict - A nested dictionary, as returned by `dandi.support.bids.schema.load_schema()`. + A nested dictionary, as returned by `schemacode.schema.load_schema()`. Notes ----- - * Couldn't find where the `label` type is defined as alphanumeric, hard-coding - `entity_definitions["subject"]["format"]`-type entries as`[a-zA-Z0-9]*?` for the time - being. - Apparently there is a `label` (alphanumeric) versus `index` (integer) specification: - https://github.com/bids-standard/bids-specification/issues/956#issuecomment-992967479 - but this is not yet used in the YAML. * Suggest to BIDS-specification to remove the periods from the extensions, the leading period is not part of the extension, but a delimiter defining the fact that it's an extension. Code sections marked as `Making it period-safe` should be edited when this fix is in, @@ -266,8 +264,6 @@ def load_entities( A list of dictionaries, with keys including 'regex' and 'mandatory'. """ - label = "([a-zA-Z0-9]*?)" - # Parsing tabular_metadata as a datatype, might be done automatically if the YAML is moved # to the same subdirectory my_schema["rules"]["datatypes"]["tabular_metadata"] = my_schema["rules"][ @@ -276,6 +272,8 @@ def load_entities( datatypes = my_schema["rules"]["datatypes"] entity_order = my_schema["rules"]["entities"] entity_definitions = my_schema["objects"]["entities"] + formats = my_schema["objects"]["formats"] + # Descriptions are not needed and very large. for i in entity_definitions.values(): i.pop("description", None) @@ -290,7 +288,7 @@ def load_entities( regex_schema = [] for datatype in datatypes: - for variant in datatypes[datatype]: + for variant in datatypes[datatype].values(): regex_entities = "" for entity in entity_order: # Slightly awkward construction to account for new-style file specification. @@ -298,7 +296,7 @@ def load_entities( # https://github.com/bids-standard/bids-specification/pull/987 try: if entity in variant["entities"]: - entity_shorthand = entity_definitions[entity]["entity"] + entity_shorthand = entity_definitions[entity]["name"] if "enum" in entity_definitions[entity].keys(): # Entity key-value pattern with specific allowed values # tested, works! @@ -306,7 +304,10 @@ def load_entities( "|".join(entity_definitions[entity]["enum"]), ) else: - variable_field = label + format_selection = formats[ + entity_definitions[entity]["format"] + ] + variable_field = f"({format_selection['pattern']})" regex_entities = _add_entity( regex_entities, entity, @@ -320,7 +321,12 @@ def load_entities( regex_string = _add_suffixes(regex_entities, variant) regex_string = _add_extensions(regex_string, variant) regex_string = _add_subdirs( - regex_string, variant, datatype, entity_definitions, modality_datatypes + regex_string, + variant, + datatype, + entity_definitions, + formats, + modality_datatypes, ) regex_string = f".*?{regex_string}$" @@ -349,6 +355,8 @@ def load_all( ------- all_regex : list of dict A list of dictionaries, with keys including 'regex' and 'mandatory'. + my_schema : list of dict + Nested dictionaries representing the full schema. """ my_schema = schema.load_schema(schema_dir) @@ -360,13 +368,14 @@ def load_all( ) all_regex.extend(top_level_regex) - return all_regex + return all_regex, my_schema def validate_all( bids_paths, regex_schema, debug=False, + pseudofile_suffixes=[], ): """ Validate `bids_paths` based on a `regex_schema` dictionary list, including regexes. @@ -380,6 +389,11 @@ def validate_all( debug : tuple, optional Whether to print itemwise notices for checks on the console, and include them in the validation result. + pseudofile_suffixes : list of str, optional + Any suffixes which identify BIDS-valid directory data. + These pseudo-file suffixes will be validated based on the directory name, with the + directory contents not being indexed for validation. + By default, no pseudo-file suffixes are checked. Returns ------- @@ -398,7 +412,7 @@ def validate_all( """ tracking_schema = deepcopy(regex_schema) - paths_list = _get_paths(bids_paths) + paths_list = _get_paths(bids_paths, pseudofile_suffixes=pseudofile_suffixes) tracking_paths = deepcopy(paths_list) if debug: itemwise_results = [] @@ -453,7 +467,7 @@ def validate_all( def write_report( validation_result, - report_path="{logdir}/bids-validator-report_{datetime}-{pid}.log", + report_path="/var/tmp/bids-validator/report_{datetime}-{pid}.log", datetime_format="%Y%m%d%H%M%SZ", ): """Write a human-readable report based on the validation result. @@ -466,10 +480,10 @@ def write_report( The "itemwise" value, if present, should be a list of dictionaries, with keys including "path", "regex", and "match". report_path : str, optional - A path under which the report is to be saved, `datetime`, `logdir`, and `pid` + A path under which the report is to be saved, `datetime`, and `pid` are available as variables for string formatting, and will be expanded to the - current datetime (as per the `datetime_format` parameter), application log - directory, and process ID, respectively. + current datetime (as per the `datetime_format` parameter) + and process ID, respectively. datetime_format : str, optional A datetime format, optionally used for the report path. @@ -478,10 +492,7 @@ def write_report( * Not using f-strings in order to prevent arbitrary code execution. """ - logdir = appdirs.user_log_dir("dandi-cli", "dandi") - report_path = report_path.format( - logdir=logdir, datetime=datetime.datetime.utcnow().strftime(datetime_format), pid=os.getpid(), ) @@ -501,8 +512,8 @@ def write_report( else: comparison_result = "no match" f.write( - f'- Comparing the `{comparison["path"]}` path to the `{comparison["regex"]}` "\ - resulted in {comparison_result}.\n' + f'- Comparing the `{comparison["path"]}` path to the `{comparison["regex"]}` ' + f"pattern resulted in {comparison_result}.\n" ) except KeyError: pass @@ -513,7 +524,7 @@ def write_report( for regex_entry in validation_result["schema_listing"]: f.write(f'\n\t- `{regex_entry["regex"]}`') f.write("\n") - if validation_result["path_tracking"]: + if len(validation_result["path_tracking"]) > 0: f.write("The following files were not matched by any regex schema entry:") f.write("\n\t* `") f.write("`\n\t* `".join(validation_result["path_tracking"])) @@ -548,7 +559,7 @@ def select_schema_dir( bids_paths, schema_reference_root, schema_version, - schema_min_version="1.7.0+012+dandi001", + schema_min_version="1.7.0+369", ): """ Select schema directory, according to a fallback logic whereby the schema path is @@ -593,6 +604,7 @@ def select_schema_dir( schema_reference_root = os.path.abspath(os.path.expanduser(schema_reference_root)) if schema_version: if "/" in schema_version: + schema_dir = schema_version if schema_version.startswith("{module_path}"): schema_dir = schema_version.format(module_path=module_path) schema_dir = os.path.abspath(os.path.expanduser(schema_dir)) @@ -689,12 +701,42 @@ def log_errors(validation_result): lgr.warning("The `%s` file was not matched by any regex schema entry.", i) +def _get_directory_suffixes(my_schema): + """Query schema for suffixes which identify directory entities. + + Parameters + ---------- + my_schema : dict + Nested directory as produced by `schemacode.schema.load_schema()`. + + Returns + ------- + list of str + Directory pseudofile suffixes excluding trailing slashes. + + Notes + ----- + * Yes this seems super-awkward to do explicitly, after all, the trailing slash is + already in so it should automagically work, but no: + - Subdirectory names need to be dynamically excluded from validation input. + - Backslash directory delimiters are still in use, which is regrettable. + """ + pseudofile_suffixes = [] + for i in my_schema["objects"]["extensions"].values(): + i_value = i["value"] + if i_value.endswith("/"): + if i_value != "/": + pseudofile_suffixes.append(i_value[:-1]) + return pseudofile_suffixes + + def validate_bids( bids_paths, schema_reference_root="{module_path}/support/bids/schemadata/", schema_version=None, debug=False, report_path=False, + suppress_errors=False, ): """ Validate paths according to BIDS schema. @@ -734,6 +776,12 @@ def validate_bids( >>> bids_paths = '~/.data2/datalad/000026/rawdata' >>> schema_version='{module_path}/data/schema/' >>> validator.validate_bids(bids_paths, schema_version=schema_version, debug=False)" + + Notes + ----- + * Needs to account for inheritance principle, probably somewhere deeper in the logic, might be + as simple as pattern parsing and multiplying patterns to which inheritance applies. + https://github.com/bids-standard/bids-specification/pull/969#issuecomment-1132119492 """ if isinstance(bids_paths, str): @@ -742,25 +790,16 @@ def validate_bids( bids_schema_dir = select_schema_dir( bids_paths, schema_reference_root, schema_version ) - regex_schema = load_all(bids_schema_dir) + regex_schema, my_schema = load_all(bids_schema_dir) + pseudofile_suffixes = _get_directory_suffixes(my_schema) validation_result = validate_all( bids_paths, regex_schema, debug=debug, + pseudofile_suffixes=pseudofile_suffixes, ) - # Record schema version. - # Not sure whether to incorporate in validation_result. - if bids_schema_dir == os.path.join( - os.path.abspath(os.path.dirname(__file__)), - "data", - "schema", - ): - # Declare we are using live version, - # string will evaluate as larger than numbered versions. - schema_version = "99999.0.0" - else: - _, schema_version = os.path.split(bids_schema_dir) - validation_result["bids_schema_version"] = schema_version + + log_errors(validation_result) if report_path: if isinstance(report_path, str): @@ -768,6 +807,4 @@ def validate_bids( else: write_report(validation_result) - log_errors(validation_result) - return validation_result diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index 7a8b21998..5120a7d0d 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -226,7 +226,7 @@ def fixture() -> Iterator[str]: nwb_test_data = get_gitrepo_fixture("http://github.com/dandi-datasets/nwb_test_data") -bids_examples = get_gitrepo_fixture("https://github.com/dandi/bids-examples") +bids_examples = get_gitrepo_fixture("https://github.com/bids-standard/bids-examples") LOCAL_DOCKER_DIR = Path(__file__).with_name("data") / "dandiarchive-docker" LOCAL_DOCKER_ENV = LOCAL_DOCKER_DIR.name diff --git a/dandi/tests/test_bids_validator_xs.py b/dandi/tests/test_bids_validator_xs.py index 42b13071b..56c9d1d70 100644 --- a/dandi/tests/test_bids_validator_xs.py +++ b/dandi/tests/test_bids_validator_xs.py @@ -1,22 +1,19 @@ import os -BIDS_EXAMPLES_BLACKLIST = [ - "invalid_pet001", -] BIDS_EXAMPLES_WHITELIST = [ "asl003", "eeg_cbm", "hcp_example_bids", "micr_SEM", - "micr_SEM-dandi", + "micr_SEMzarr", "micr_SPIM", "pet001", "pet003", - "qmri_megre", "qmri_tb1tfl", - "qmri_vfa", + "qmri_vfa/derivatives/qMRLab", ] -TEST_SCHEMA_PATH = "{module_path}/support/bids/schemadata/1.7.0+012+dandi001" + +TEST_SCHEMA_PATH = "{module_path}/support/bids/schemadata/1.7.0+369" def test__add_entity(): @@ -133,24 +130,29 @@ def test__add_subdirs(): datatype = "tabular_metadata" entity_definitions = { "acquisition": { - "name": "Acquisition", - "entity": "acq", + "display_name": "Acquisition", + "name": "acq", "type": "string", "format": "label", }, "session": { - "name": "Session", - "entity": "ses", + "display_name": "Session", + "name": "ses", "type": "string", "format": "label", }, "subject": { - "name": "Subject", - "entity": "sub", + "display_name": "Subject", + "name": "sub", "type": "string", "format": "label", }, } + formats = { + "label": { + "pattern": "[0-9a-zA-Z]+", + } + } modality_datatypes = [ "anat", "dwi", @@ -165,11 +167,11 @@ def test__add_subdirs(): "micr", ] _regex_string = _add_subdirs( - regex_string, variant, datatype, entity_definitions, modality_datatypes + regex_string, variant, datatype, entity_definitions, formats, modality_datatypes ) assert ( - _regex_string == "/sub-(?P([a-zA-Z0-9]*?))/sub-(?P=subject)" + _regex_string == "/sub-(?P([0-9a-zA-Z]+))/sub-(?P=subject)" "_sessions\\.(tsv|json)" ) @@ -239,9 +241,10 @@ def test_load_all(): schema_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), - "../support/bids/schemadata/1.7.0+012+dandi001", + "../support/bids/schemadata/1.7.0+369", ) - schema_all = load_all(schema_path) + + schema_all, _ = load_all(schema_path) # Check if expected keys are present in all entries for entry in schema_all: @@ -307,35 +310,21 @@ def test_write_report(tmp_path): assert report_text == expected_report_text -def test_bids_datasets(bids_examples): +def test_bids_datasets(bids_examples, tmp_path): from dandi.bids_validator_xs import validate_bids - # Validate per dataset, with automatic schema selection: + # Validate per dataset with explicit schema specification: for i in os.listdir(bids_examples): if i in BIDS_EXAMPLES_WHITELIST: result = validate_bids( os.path.join(bids_examples, i), + schema_version=TEST_SCHEMA_PATH, + report_path=True, + debug=True, ) # Have all files been validated? assert len(result["path_tracking"]) == 0 - -def test_error_datasets(bids_examples): - from dandi.bids_validator_xs import validate_bids - - # Validate per dataset, with automatic schema selection: - for i in os.listdir(bids_examples): - if i in BIDS_EXAMPLES_BLACKLIST: - result = validate_bids( - os.path.join(bids_examples, i), - ) - # Are there non-validated files? - assert len(result["path_tracking"]) != 0 - - -def test_bids_datasets_selected_paths(bids_examples, tmp_path): - from dandi.bids_validator_xs import validate_bids - # Create input for file list based validation selected_dir = os.path.join(bids_examples, BIDS_EXAMPLES_WHITELIST[0]) selected_paths = [] @@ -343,19 +332,18 @@ def test_bids_datasets_selected_paths(bids_examples, tmp_path): for f in files: selected_path = os.path.join(root, f) selected_paths.append(selected_path) - # Does explicit schema specification work? - result = validate_bids(selected_paths, schema_version=TEST_SCHEMA_PATH) - - # Does terminal debug output work? + # Does terminal debug output and schema auto-fallback work? result = validate_bids(selected_paths, debug=True) + # Does default log path specification work? + result = validate_bids( + selected_paths, schema_version=TEST_SCHEMA_PATH, report_path=True + ) - # Does the default report path work? - result = validate_bids(selected_paths, report_path=True) - - # Does custom report path specification work? + # Does custom log path specification work? result = validate_bids( selected_paths, schema_version=TEST_SCHEMA_PATH, + debug=True, report_path=os.path.join(tmp_path, "test_bids.log"), ) # Have all files been validated? From 46988a3904a74f9182208f6ef82958bb1461a331 Mon Sep 17 00:00:00 2001 From: Horea Christian Date: Fri, 8 Jul 2022 00:03:54 -0400 Subject: [PATCH 02/28] Schema update --- .../1.7.0+012/rules/associated_data.yaml | 11 - .../1.7.0+012/rules/tabular_metadata.yaml | 20 - .../{1.7.0+012 => 1.7.0+369}/README.md | 67 ++- .../bids/schemadata/1.7.0+369/SCHEMA_VERSION | 1 + .../schemadata/1.7.0+369/meta/context.yaml | 262 +++++++++ .../objects/associated_data.yaml | 20 +- .../objects/columns.yaml | 140 ++++- .../objects/datatypes.yaml | 33 +- .../objects/entities.yaml | 143 ++--- .../1.7.0+369/objects/extensions.yaml | 299 +++++++++++ .../schemadata/1.7.0+369/objects/formats.yaml | 124 +++++ .../objects/metadata.yaml | 496 +++++++++++++++--- .../objects/modalities.yaml | 14 +- .../objects/suffixes.yaml | 304 +++++++---- .../objects/top_level_files.yaml | 24 +- .../1.7.0+369/rules/associated_data.yaml | 11 + .../1.7.0+369/rules/checks/asl.yaml | 279 ++++++++++ .../1.7.0+369/rules/checks/dwi.yaml | 61 +++ .../1.7.0+369/rules/checks/events.yaml | 16 + .../1.7.0+369/rules/checks/fmap.yaml | 26 + .../1.7.0+369/rules/checks/func.yaml | 16 + .../rules/common_derivatives_validation.yaml | 18 + .../1.7.0+369/rules/dataset_metadata.yaml | 67 +++ .../rules/datatypes/anat.yaml | 54 +- .../rules/datatypes/beh.yaml | 11 +- .../derivatives/common_derivatives.yaml | 413 +++++++++++++++ .../common_imaging_derivatives.yaml | 220 ++++++++ .../rules/datatypes/dwi.yaml | 21 +- .../rules/datatypes/eeg.yaml | 38 +- .../rules/datatypes/fmap.yaml | 42 +- .../rules/datatypes/func.yaml | 24 +- .../rules/datatypes/ieeg.yaml | 36 +- .../rules/datatypes/meg.yaml | 70 ++- .../rules/datatypes/micr.yaml | 12 +- .../rules/datatypes/perf.yaml | 24 +- .../rules/datatypes/pet.yaml | 24 +- .../rules/entities.yaml | 0 .../rules/modalities.yaml | 0 .../1.7.0+369/rules/sidecars/anat.yaml | 34 ++ .../1.7.0+369/rules/sidecars/asl.yaml | 285 ++++++++++ .../1.7.0+369/rules/sidecars/beh.yaml | 21 + .../1.7.0+369/rules/sidecars/continuous.yaml | 26 + .../derivatives/common_derivatives.yaml | 85 +++ .../1.7.0+369/rules/sidecars/dwi.yaml | 25 + .../1.7.0+369/rules/sidecars/eeg.yaml | 166 ++++++ .../1.7.0+369/rules/sidecars/entities.yaml | 81 +++ .../1.7.0+369/rules/sidecars/fmap.yaml | 87 +++ .../1.7.0+369/rules/sidecars/func.yaml | 110 ++++ .../1.7.0+369/rules/sidecars/ieeg.yaml | 144 +++++ .../1.7.0+369/rules/sidecars/meg.yaml | 261 +++++++++ .../1.7.0+369/rules/sidecars/micr.yaml | 79 +++ .../1.7.0+369/rules/sidecars/mri.yaml | 263 ++++++++++ .../1.7.0+369/rules/sidecars/pet.yaml | 183 +++++++ .../derivatives/common_derivatives.yaml | 11 + .../1.7.0+369/rules/tabular_data/eeg.yaml | 43 ++ .../1.7.0+369/rules/tabular_data/ieeg.yaml | 49 ++ .../1.7.0+369/rules/tabular_data/meg.yaml | 23 + .../1.7.0+369/rules/tabular_data/perf.yaml | 8 + .../1.7.0+369/rules/tabular_data/pet.yaml | 52 ++ .../1.7.0+369/rules/tabular_data/physio.yaml | 9 + .../1.7.0+369/rules/tabular_data/task.yaml | 15 + .../1.7.0+369/rules/tabular_metadata.yaml | 18 + .../rules/top_level_files.yaml | 3 + 63 files changed, 5064 insertions(+), 458 deletions(-) delete mode 100644 dandi/support/bids/schemadata/1.7.0+012/rules/associated_data.yaml delete mode 100644 dandi/support/bids/schemadata/1.7.0+012/rules/tabular_metadata.yaml rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/README.md (86%) create mode 100644 dandi/support/bids/schemadata/1.7.0+369/SCHEMA_VERSION create mode 100644 dandi/support/bids/schemadata/1.7.0+369/meta/context.yaml rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/associated_data.yaml (50%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/columns.yaml (81%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/datatypes.yaml (61%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/entities.yaml (84%) create mode 100644 dandi/support/bids/schemadata/1.7.0+369/objects/extensions.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/objects/formats.yaml rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/metadata.yaml (84%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/modalities.yaml (62%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/suffixes.yaml (79%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/objects/top_level_files.yaml (80%) create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/associated_data.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/checks/asl.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/checks/dwi.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/checks/events.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/checks/fmap.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/checks/func.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/common_derivatives_validation.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/dataset_metadata.yaml rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/anat.yaml (86%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/beh.yaml (81%) create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/datatypes/derivatives/common_derivatives.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/datatypes/derivatives/common_imaging_derivatives.yaml rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/dwi.yaml (75%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/eeg.yaml (78%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/fmap.yaml (85%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/func.yaml (86%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/ieeg.yaml (79%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/meg.yaml (78%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/micr.yaml (83%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/perf.yaml (82%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/datatypes/pet.yaml (83%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/entities.yaml (100%) rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/modalities.yaml (100%) create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/anat.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/asl.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/beh.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/continuous.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/derivatives/common_derivatives.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/dwi.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/eeg.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/entities.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/fmap.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/func.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/ieeg.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/meg.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/micr.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/mri.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/sidecars/pet.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/derivatives/common_derivatives.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/eeg.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/ieeg.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/meg.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/perf.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/pet.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/physio.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_data/task.yaml create mode 100644 dandi/support/bids/schemadata/1.7.0+369/rules/tabular_metadata.yaml rename dandi/support/bids/schemadata/{1.7.0+012 => 1.7.0+369}/rules/top_level_files.yaml (93%) diff --git a/dandi/support/bids/schemadata/1.7.0+012/rules/associated_data.yaml b/dandi/support/bids/schemadata/1.7.0+012/rules/associated_data.yaml deleted file mode 100644 index 609fcafaa..000000000 --- a/dandi/support/bids/schemadata/1.7.0+012/rules/associated_data.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# This file describes the requirement levels of folders which may appear -# within a dataset folder without following BIDS format. -code: - required: false -derivatives: - required: false -sourcedata: - required: false -stimuli: - required: false diff --git a/dandi/support/bids/schemadata/1.7.0+012/rules/tabular_metadata.yaml b/dandi/support/bids/schemadata/1.7.0+012/rules/tabular_metadata.yaml deleted file mode 100644 index e04d10c1f..000000000 --- a/dandi/support/bids/schemadata/1.7.0+012/rules/tabular_metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# scans.tsv -- suffixes: - - scans - extensions: - - .tsv - - .json - entities: - subject: required - # session is required if session is present in the dataset. - session: optional -# sessions.tsv -# This file may only exist if session is present in the dataset. -- suffixes: - - sessions - extensions: - - .tsv - - .json - entities: - subject: required diff --git a/dandi/support/bids/schemadata/1.7.0+012/README.md b/dandi/support/bids/schemadata/1.7.0+369/README.md similarity index 86% rename from dandi/support/bids/schemadata/1.7.0+012/README.md rename to dandi/support/bids/schemadata/1.7.0+369/README.md index c3d142cba..81e3b9d63 100644 --- a/dandi/support/bids/schemadata/1.7.0+012/README.md +++ b/dandi/support/bids/schemadata/1.7.0+369/README.md @@ -1,9 +1,9 @@ # BIDS-schema -Portions of the BIDS specification are defined using YAML files, in order to +Portions of the BIDS specification are defined using YAML files in order to make the specification machine-readable. -Currently, the portions of the specification that rely on this schema are +Currently the portions of the specification that rely on this schema are the entity tables, entity definitions, filename templates, and metadata tables. Any changes to the specification should be mirrored in the schema. @@ -32,14 +32,14 @@ The types of objects currently supported in the schema are: - suffixes, - metadata, - top-level files, -- and non-BIDS associated folders. +- and non-BIDS associated directories. -Each of these object types has a single file in the `objects/` folder. +Each of these object types has a single file in the `objects/` directory. - `modalities.yaml`: The modalities, or types of technology, used to acquire data in a BIDS dataset. These modalities are not reflected directly in the specification. For example, while both fMRI and DWI data are acquired with an MRI, - in a BIDS dataset they are stored in different folders reflecting the two different `datatypes`. + in a BIDS dataset they are stored in different directories reflecting the two different `datatypes`. - `datatypes.yaml`: Data types supported by the specification. The only information provided in the file is: @@ -48,7 +48,7 @@ Each of these object types has a single file in the `objects/` folder. 1. each datatype's full name 1. a free text description of the datatype. -- `entities.yaml`: Entities (key/value pairs in folder and filenames). +- `entities.yaml`: Entities (key-value pairs in directory and filenames). - `metadata.yaml`: All valid metadata fields that are explicitly supported in BIDS sidecar JSON files. @@ -58,7 +58,7 @@ Each of these object types has a single file in the `objects/` folder. - `top_level_files.yaml`: Valid top-level files which may appear in a BIDS dataset. -- `associated_data.yaml`: Folders that may appear within a dataset folder without following BIDS rules. +- `associated_data.yaml`: Directories that may appear within a dataset directory without following BIDS rules. ### On re-used objects with different definitions @@ -73,7 +73,7 @@ For objects with `snake_case` names, two underscores must be used. There should also be a comment near the object definition in the YAML file describing the nature of the different objects. For example, the TSV column `"reference"` means different things when used for EEG data, as compared to iEEG data. -As such, there are two definitions in `columns.yaml` for the `"reference"` column: `"reference__eeg"` and `"reference_ieeg"`. +As such, there are two definitions in `columns.yaml` for the `"reference"` column: `"reference__eeg"` and `"reference__ieeg"`. ```yaml # reference column for channels.tsv files for EEG data @@ -115,7 +115,7 @@ The `description` field is a freeform description of the modality. ### `datatypes.yaml` This file contains a dictionary in which each datatype is defined. -Keys are the folder names associated with each datatype (for example, `anat` for anatomical MRI), +Keys are the directory names associated with each datatype (for example, `anat` for anatomical MRI), and each associated value is a dictionary with two keys: `name` and `description`. The `name` field is the full name of the datatype. @@ -123,7 +123,7 @@ The `description` field is a freeform description of the datatype. ### `entities.yaml` -This file contains a dictionary in which each entity (key/value pair in filenames) is defined. +This file contains a dictionary in which each entity (key-value pair in filenames) is defined. Keys are long-form versions of the entities, which are distinct from both the entities as they appear in filenames _and_ their full names. For example, the key for the "Contrast Enhancing Agent" entity, which appears in filenames as `ce-