From e7b7c376da19f95745e5f1240fcaa432a4d29a6b Mon Sep 17 00:00:00 2001 From: David JULIEN Date: Thu, 16 Jul 2020 11:41:26 +0200 Subject: [PATCH 01/30] [fNIRS]feat: add nosatflags_wlX files support When the probes get saturated (i.e. when the value they return is "too high"), NIRStar replaces the value with 'NaN' in the standard .wlX file. The true value is kept in a .nosatflags_wlX file, which is a copy of a .wlX file with the true probed values. It is unclear how NIRstar decides that a probe got satured, so we added a flag to `read_raw_nirx()` and `RawNIRX.__init__` to let the user chose which file to use. Providing two *.wl1 or two *.wl2 files still raises an error, as we check for the existence and unicity of the corresponding *.nosatflags_wlX file. Note that the directory structure check doesn't yet discriminate between *.wlX and *.nosatflags_wlX (as it checks for any file ending with 'wlX'). Changing the check could be done in another PR to make sure the user did not provide only *.nosatflags_wlX files and maybe warn them. --- mne/io/nirx/nirx.py | 82 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 88d64987e90..db77d84b242 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -20,15 +20,17 @@ @fill_doc -def read_raw_nirx(fname, preload=False, verbose=None): +def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): """Reader for a NIRX fNIRS recording. - This function has only been tested with NIRScout devices. - Parameters ---------- fname : str Path to the NIRX data folder or header file. + saturated : str + (Only relevant for NIRSport1 devices). If 'ignore' (default), + use *.nosatflags_wlX instead of standard *.wlX files. If 'nan', + use standard *.wlX files. Irrelevant if there is no *.nosatflags file. %(preload)s %(verbose)s @@ -40,8 +42,22 @@ def read_raw_nirx(fname, preload=False, verbose=None): See Also -------- mne.io.Raw : Documentation of attribute and methods. + + Notes + ----- + This function has only been tested with NIRScout and NIRSport1 devices. + + - Re: saturated flag + The NIRSport probes can saturate during the experiment. Starting from + NIRStar 14.2, those saturated values are replaced by NaN in the + standard *.wlX files. The measured values are stored in another file + called *.nosatflags_wlX, which is a copy of the corresponding *.wlX + file where the saturated data didn't get replaced. Since NaN values can + cause unexpected behaviour with mathematical functions, you can chose + to use the original, non-modified data by setting the ``saturated`` flag to + 'ignore' (default) or set it to 'nan' to use NaN values. """ - return RawNIRX(fname, preload, verbose) + return RawNIRX(fname, saturated, preload, verbose) def _open(fname): @@ -56,16 +72,36 @@ class RawNIRX(BaseRaw): ---------- fname : str Path to the NIRX data folder or header file. + saturated : str + (Only relevant for NIRSport1 devices). If 'ignore' (default), + use *.nosatflags_wlX instead of standard *.wlX files. If 'nan', + use standard *.wlX files. Irrelevant if there is no *.nosatflags file. + %(preload)s %(verbose)s See Also -------- mne.io.Raw : Documentation of attribute and methods. + + Notes + ----- + This function has only been tested with NIRScout and NIRSport1 devices. + + - Re: saturated flag + The NIRSport probes can saturate during the experiment. Starting from + NIRStar 14.2, those saturated values are replaced by NaN in the + standard *.wlX files. The measured values are stored in another file + called *.nosatflags_wlX, which is a copy of the corresponding *.wlX + file where the saturated data didn't get replaced. Since NaN values can + cause unexpected behaviour with mathematical functions, you can chose + to use the original, non-modified data by setting the ``saturated`` flag of + the read_raw_nirx() method to 'ignore' (default) or set it to 'nan' to + use NaN values. """ @verbose - def __init__(self, fname, preload=False, verbose=None): + def __init__(self, fname, saturated, preload=False, verbose=None): from ...externals.pymatreader import read_mat from ...coreg import get_mni_fiducials # avoid circular import prob logger.info('Loading %s' % fname) @@ -83,9 +119,36 @@ def __init__(self, fname, preload=False, verbose=None): for key in keys: files[key] = glob.glob('%s/*%s' % (fname, key)) if len(files[key]) != 1: - raise RuntimeError('Expect one %s file, got %d' % - (key, len(files[key]),)) - files[key] = files[key][0] + if (key == 'wl1' or key == 'wl2') \ + and len(glob.glob('%s/*%s' % + (fname, 'nosatflags_' + key))) == 1: + if saturated == 'nan': + warn('You provided saturated data and specified ' + 'to use the standard *.wlX files.') + files[key] = files[key][1] + else: + if saturated == 'ignore': + warn('The data you provided contains NaN entries ' + 'which were put by NIRStar in the *.wlX ' + 'files. You chose to ignore them and use the ' + '*.nosatflags_wlX files instead. You can ' + 'change this behaviour by setting ' + '``saturated`` to "nan" when calling ' + 'read_raw_nirx()') + else: + warn('The value specified for ``saturated`` is ' + 'not recognized. Falling back to default ' + 'behaviour and using *.nosatflags_wlX files. ' + 'You can change this behaviour by setting ' + '``saturated`` to "nan" when calling ' + 'read_raw_nirx()') + files[key] = glob.glob('%s/*%s' % + (fname, 'nosatflags_' + key))[0] + else: + raise RuntimeError('Expect one %s file, got %d' % + (key, len(files[key]),)) + else: + files[key] = files[key][0] if len(glob.glob('%s/*%s' % (fname, 'dat'))) != 1: warn("A single dat file was expected in the specified path, but " "got %d. This may indicate that the file structure has been " @@ -111,7 +174,8 @@ def __init__(self, fname, preload=False, verbose=None): if hdr['GeneralInfo']['NIRStar'] not in ['"15.0"', '"15.2"', '"15.3"']: raise RuntimeError('MNE does not support this NIRStar version' ' (%s)' % (hdr['GeneralInfo']['NIRStar'],)) - if "NIRScout" not in hdr['GeneralInfo']['Device']: + if "NIRScout" not in hdr['GeneralInfo']['Device'] \ + and "NIRSport" not in hdr['GeneralInfo']['Device']: warn("Only import of data from NIRScout devices have been " "thoroughly tested. You are using a %s device. " % hdr['GeneralInfo']['Device']) From 06f46c0830cb545afce1da6f302e07e652ce5f34 Mon Sep 17 00:00:00 2001 From: David JULIEN Date: Thu, 2 Jul 2020 11:31:44 +0200 Subject: [PATCH 02/30] [fNIRS]feat: add nosatflags tests for NIRSport1 --- mne/io/nirx/tests/test_nirx.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 37f329c3212..2182ab1428b 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -72,6 +72,22 @@ def test_nirx_dat_warn(tmpdir): read_raw_nirx(fname, preload=True) +@requires_testing_data +def test_nirx_nosatflags_v1_warn(tmpdir): + """Test reading NIRSportv1 files with saturated data.""" + shutil.copytree(fname_nirx_15_2_short, str(tmpdir) + "/data/") + shutil.copyfile(str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.wl1", + str(tmpdir) + "/data" + + "/NIRS-2019-08-23_001.nosatflags_wl1") + fname = str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.hdr" + with pytest.raises(RuntimeWarning, match='specified to use the standard'): + read_raw_nirx(fname, saturated='nan', preload=True) + with pytest.raises(RuntimeWarning, match='You chose to ignore them'): + read_raw_nirx(fname, saturated='ignore', preload=True) + with pytest.raises(RuntimeWarning, match='Falling back to default'): + read_raw_nirx(fname, saturated='foobar', preload=True) + + @requires_testing_data def test_nirx_15_2_short(): """Test reading NIRX files.""" From 8e73ca4e72913d9bb721baf70b7f77ce90a42bc4 Mon Sep 17 00:00:00 2001 From: David JULIEN Date: Wed, 26 Aug 2020 10:16:13 +0200 Subject: [PATCH 03/30] [fNIRS]feat: add 'NaN' annotations Some devices register 'NaN' values when the probes saturate. Since 'NaN' values can cause unexpected behaviour, we added the ability to annotate them so that they can be filtered out during the process. --- doc/python_reference.rst | 1 + mne/io/nirx/nirx.py | 9 ++++++ mne/preprocessing/__init__.py | 1 + mne/preprocessing/annotate_nan.py | 47 +++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 mne/preprocessing/annotate_nan.py diff --git a/doc/python_reference.rst b/doc/python_reference.rst index be7f114f6bb..45404145768 100644 --- a/doc/python_reference.rst +++ b/doc/python_reference.rst @@ -364,6 +364,7 @@ Projections: annotate_flat annotate_movement annotate_muscle_zscore + annotate_nan compute_average_dev_head_t compute_current_source_density compute_fine_calibration diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index db77d84b242..6aa0c618af7 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -104,6 +104,7 @@ class RawNIRX(BaseRaw): def __init__(self, fname, saturated, preload=False, verbose=None): from ...externals.pymatreader import read_mat from ...coreg import get_mni_fiducials # avoid circular import prob + from ...preprocessing import annotate_nan # avoid circular import prob logger.info('Loading %s' % fname) if fname.endswith('.hdr'): @@ -126,6 +127,10 @@ def __init__(self, fname, saturated, preload=False, verbose=None): warn('You provided saturated data and specified ' 'to use the standard *.wlX files.') files[key] = files[key][1] + elif saturated == 'annotate': + warn('You provided saturated data and specified ' + 'to annotate your data with a \'nan\' flag.') + files[key] = files[key][1] else: if saturated == 'ignore': warn('The data you provided contains NaN entries ' @@ -384,6 +389,10 @@ def prepend(li, str): annot = Annotations(onset, duration, description) self.set_annotations(annot) + if saturated == "annotate": + annot = annotate_nan(self) + self.set_annotations(annot) + def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): """Read a segment of data from a file. diff --git a/mne/preprocessing/__init__.py b/mne/preprocessing/__init__.py index dddf063846c..30de8ce02b8 100644 --- a/mne/preprocessing/__init__.py +++ b/mne/preprocessing/__init__.py @@ -28,3 +28,4 @@ from ._regress import regress_artifact from ._fine_cal import (compute_fine_calibration, read_fine_calibration, write_fine_calibration) +from .annotate_nan import annotate_nan diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py new file mode 100644 index 00000000000..8bfb3d2ef47 --- /dev/null +++ b/mne/preprocessing/annotate_nan.py @@ -0,0 +1,47 @@ +# Author: David Julien +# +# License: BSD (3-clause) + +import numpy as np + +import warnings + +from ..annotations import Annotations +from ..utils import _mask_to_onsets_offsets + + +def annotate_nan(raw): + """Detect segments with NaN and return a new Annotations instance. + + Parameters + ---------- + raw : instance of Raw + Data to find segments with NaN values. + + Returns + ------- + annot : instance of Annotations + Updated annotations for raw data. + """ + annot = raw.annotations.copy() + data, times = raw.get_data(return_times=True) + sampling_duration = 1 / raw.info['sfreq'] + + nans = np.any(np.isnan(data), axis=0) + starts, stops = _mask_to_onsets_offsets(nans) + + if len(starts) > 0: + starts, stops = np.array(starts), np.array(stops) + onsets = (starts + raw.first_samp) * sampling_duration + durations = (stops - starts) * sampling_duration + else: + warnings.warn("The dataset you provided does not contain 'NaN' " + "values. No annotation were made.") + return + + if annot is None: + annot = Annotations(onsets, durations, 'bad_NAN') + else: + annot.append(onsets, durations, 'bad_NAN') + + return annot From 184c91e1306af3696bc8f9e335fbd569080ca95d Mon Sep 17 00:00:00 2001 From: David JULIEN Date: Wed, 26 Aug 2020 10:16:33 +0200 Subject: [PATCH 04/30] [fNIRS]feat: add a test for data with annotated_nan --- mne/io/nirx/tests/test_nirx.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 2182ab1428b..0547969bb2b 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -82,6 +82,8 @@ def test_nirx_nosatflags_v1_warn(tmpdir): fname = str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.hdr" with pytest.raises(RuntimeWarning, match='specified to use the standard'): read_raw_nirx(fname, saturated='nan', preload=True) + with pytest.raises(RuntimeWarning, match='specified to annotate your'): + read_raw_nirx(fname, saturated='annotate', preload=True) with pytest.raises(RuntimeWarning, match='You chose to ignore them'): read_raw_nirx(fname, saturated='ignore', preload=True) with pytest.raises(RuntimeWarning, match='Falling back to default'): From aa773c23b68707fbdabb94c26c8da3b087ce4817 Mon Sep 17 00:00:00 2001 From: Romain Derollepot Date: Wed, 9 Dec 2020 16:09:42 +0100 Subject: [PATCH 05/30] [fNIRS] update release number and hash following the addition of NIRSport v1 testing data --- mne/datasets/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/datasets/utils.py b/mne/datasets/utils.py index 7b5febcdfa2..c11fc7a1e30 100644 --- a/mne/datasets/utils.py +++ b/mne/datasets/utils.py @@ -245,7 +245,7 @@ def _data_path(path=None, force_update=False, update_path=True, download=True, path = _get_path(path, key, name) # To update the testing or misc dataset, push commits, then make a new # release on GitHub. Then update the "releases" variable: - releases = dict(testing='0.110', misc='0.7') + releases = dict(testing='0.111', misc='0.7') # And also update the "md5_hashes['testing']" variable below. # To update any other dataset, update the data archive itself (upload # an updated version) and update the md5 hash. @@ -331,7 +331,7 @@ def _data_path(path=None, force_update=False, update_path=True, download=True, sample='12b75d1cb7df9dfb4ad73ed82f61094f', somato='32fd2f6c8c7eb0784a1de6435273c48b', spm='9f43f67150e3b694b523a21eb929ea75', - testing='c4cd3385f321cd1151ed9de34fc4ce5a', + testing='e7ece4615882b99026edb76fb708a3ce', multimodal='26ec847ae9ab80f58f204d09e2c08367', fnirs_motor='c4935d19ddab35422a69f3326a01fef8', opm='370ad1dcfd5c47e029e692c85358a374', From 189cf516834df41b2b4efaf7d91009e8bc05c6cd Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 15:35:35 +1000 Subject: [PATCH 06/30] Add to docs --- doc/preprocessing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/preprocessing.rst b/doc/preprocessing.rst index ffb09b88904..b13c7023b25 100644 --- a/doc/preprocessing.rst +++ b/doc/preprocessing.rst @@ -71,6 +71,7 @@ Projections: annotate_flat annotate_movement annotate_muscle_zscore + annotate_nan compute_average_dev_head_t compute_current_source_density compute_fine_calibration From 258a9ecd8906f4d22d0c4661be814f393af4fa33 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 15:59:23 +1000 Subject: [PATCH 07/30] Improve wording and errors --- mne/io/nirx/nirx.py | 83 +++++++++++++++++----------------- mne/io/nirx/tests/test_nirx.py | 8 ++-- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index b3c21dacd6e..2205647e63a 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -29,9 +29,13 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): fname : str Path to the NIRX data folder or header file. saturated : str - (Only relevant for NIRSport1 devices). If 'ignore' (default), - use *.nosatflags_wlX instead of standard *.wlX files. If 'nan', - use standard *.wlX files. Irrelevant if there is no *.nosatflags file. + Replace saturated segments of data with NaNs. + If 'ignore' (default) the measured data is returned, even if it + contains measurements while the amplifier was saturated. + If 'nan' the returned data will contain NaNs during time segments + when the amplifier was saturated. + This argument will only be used if there is no *.nosatflags file + (only if a NIRSport device is used and saturation occurred). %(preload)s %(verbose)s @@ -48,15 +52,14 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): ----- This function has only been tested with NIRScout and NIRSport1 devices. - - Re: saturated flag - The NIRSport probes can saturate during the experiment. Starting from - NIRStar 14.2, those saturated values are replaced by NaN in the - standard *.wlX files. The measured values are stored in another file - called *.nosatflags_wlX, which is a copy of the corresponding *.wlX - file where the saturated data didn't get replaced. Since NaN values can - cause unexpected behaviour with mathematical functions, you can chose - to use the original, non-modified data by setting the ``saturated`` flag to - 'ignore' (default) or set it to 'nan' to use NaN values. + The NIRSport device can detect if the amplifier is saturated. + Starting from NIRStar 14.2, those saturated values are replaced by NaNs + in the standard *.wlX files. + The raw unmodified measured values are stored in another file + called *.nosatflags_wlX. As NaN values can cause unexpected behaviour with + mathematical functions the default behaviour is to return the + saturated data. However, you may request the data with saturated + segments replaced with NaN by setting the saturated argument to nan. """ return RawNIRX(fname, saturated, preload, verbose) @@ -74,9 +77,13 @@ class RawNIRX(BaseRaw): fname : str Path to the NIRX data folder or header file. saturated : str - (Only relevant for NIRSport1 devices). If 'ignore' (default), - use *.nosatflags_wlX instead of standard *.wlX files. If 'nan', - use standard *.wlX files. Irrelevant if there is no *.nosatflags file. + Replace saturated segments of data with NaNs. + If 'ignore' (default) the measured data is returned, even if it + contains measurements while the amplifier was saturated. + If 'nan' the returned data will contain NaNs during time segments + when the amplifier was saturated. + This argument will only be used if there is no *.nosatflags file + (only if a NIRSport device is used and saturation occurred). %(preload)s %(verbose)s @@ -89,16 +96,14 @@ class RawNIRX(BaseRaw): ----- This function has only been tested with NIRScout and NIRSport1 devices. - - Re: saturated flag - The NIRSport probes can saturate during the experiment. Starting from - NIRStar 14.2, those saturated values are replaced by NaN in the - standard *.wlX files. The measured values are stored in another file - called *.nosatflags_wlX, which is a copy of the corresponding *.wlX - file where the saturated data didn't get replaced. Since NaN values can - cause unexpected behaviour with mathematical functions, you can chose - to use the original, non-modified data by setting the ``saturated`` flag of - the read_raw_nirx() method to 'ignore' (default) or set it to 'nan' to - use NaN values. + The NIRSport device can detect if the amplifier is saturated. + Starting from NIRStar 14.2, those saturated values are replaced by NaNs + in the standard *.wlX files. + The raw unmodified measured values are stored in another file + called *.nosatflags_wlX. As NaN values can cause unexpected behaviour with + mathematical functions the default behaviour is to return the + saturated data. However, you may request the data with saturated + segments replaced with NaN by setting the saturated argument to nan. """ @verbose @@ -125,29 +130,23 @@ def __init__(self, fname, saturated, preload=False, verbose=None): and len(glob.glob('%s/*%s' % (fname, 'nosatflags_' + key))) == 1: if saturated == 'nan': - warn('You provided saturated data and specified ' - 'to use the standard *.wlX files.') + warn('The measurement contains saturated data. ' + 'Saturated values will be replaced by NaNs.') files[key] = files[key][1] elif saturated == 'annotate': - warn('You provided saturated data and specified ' - 'to annotate your data with a \'nan\' flag.') + warn('The measurement contains saturated data. ' + 'Saturated values will be annotated ' + 'with \'nan\' flags.') files[key] = files[key][1] else: if saturated == 'ignore': - warn('The data you provided contains NaN entries ' - 'which were put by NIRStar in the *.wlX ' - 'files. You chose to ignore them and use the ' - '*.nosatflags_wlX files instead. You can ' - 'change this behaviour by setting ' - '``saturated`` to "nan" when calling ' - 'read_raw_nirx()') + warn('The measurement contains saturated data.') else: - warn('The value specified for ``saturated`` is ' - 'not recognized. Falling back to default ' - 'behaviour and using *.nosatflags_wlX files. ' - 'You can change this behaviour by setting ' - '``saturated`` to "nan" when calling ' - 'read_raw_nirx()') + raise KeyError("The value specified for saturated " + "must be one of ignore, nan, or " + "annotate. " + f"{saturated} was provided") + files[key] = glob.glob('%s/*%s' % (fname, 'nosatflags_' + key))[0] else: diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 01d2dfcbb6b..83aebc57ab6 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -80,13 +80,13 @@ def test_nirx_nosatflags_v1_warn(tmpdir): str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.nosatflags_wl1") fname = str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.hdr" - with pytest.raises(RuntimeWarning, match='specified to use the standard'): + with pytest.raises(RuntimeWarning, match='be replaced by NaNs'): read_raw_nirx(fname, saturated='nan', preload=True) - with pytest.raises(RuntimeWarning, match='specified to annotate your'): + with pytest.raises(RuntimeWarning, match='annotated with \'nan\' flags'): read_raw_nirx(fname, saturated='annotate', preload=True) - with pytest.raises(RuntimeWarning, match='You chose to ignore them'): + with pytest.raises(RuntimeWarning, match='contains saturated data'): read_raw_nirx(fname, saturated='ignore', preload=True) - with pytest.raises(RuntimeWarning, match='Falling back to default'): + with pytest.raises(KeyError, match='specified for saturated must'): read_raw_nirx(fname, saturated='foobar', preload=True) From e2853108e09697fa4b52507e3a04f980904797db Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 16:19:22 +1000 Subject: [PATCH 08/30] Use real data for tests --- mne/io/nirx/nirx.py | 20 +++++++-------- mne/io/nirx/tests/test_nirx.py | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 2205647e63a..4d899dc3a9e 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -30,11 +30,11 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): Path to the NIRX data folder or header file. saturated : str Replace saturated segments of data with NaNs. - If 'ignore' (default) the measured data is returned, even if it + If ignore (default) the measured data is returned, even if it contains measurements while the amplifier was saturated. - If 'nan' the returned data will contain NaNs during time segments + If nan the returned data will contain NaNs during time segments when the amplifier was saturated. - This argument will only be used if there is no *.nosatflags file + This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). %(preload)s %(verbose)s @@ -54,9 +54,9 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): The NIRSport device can detect if the amplifier is saturated. Starting from NIRStar 14.2, those saturated values are replaced by NaNs - in the standard *.wlX files. + in the standard .wlX files. The raw unmodified measured values are stored in another file - called *.nosatflags_wlX. As NaN values can cause unexpected behaviour with + called .nosatflags_wlX. As NaN values can cause unexpected behaviour with mathematical functions the default behaviour is to return the saturated data. However, you may request the data with saturated segments replaced with NaN by setting the saturated argument to nan. @@ -78,11 +78,11 @@ class RawNIRX(BaseRaw): Path to the NIRX data folder or header file. saturated : str Replace saturated segments of data with NaNs. - If 'ignore' (default) the measured data is returned, even if it + If ignore (default) the measured data is returned, even if it contains measurements while the amplifier was saturated. - If 'nan' the returned data will contain NaNs during time segments + If nan the returned data will contain NaNs during time segments when the amplifier was saturated. - This argument will only be used if there is no *.nosatflags file + This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). %(preload)s @@ -98,9 +98,9 @@ class RawNIRX(BaseRaw): The NIRSport device can detect if the amplifier is saturated. Starting from NIRStar 14.2, those saturated values are replaced by NaNs - in the standard *.wlX files. + in the standard .wlX files. The raw unmodified measured values are stored in another file - called *.nosatflags_wlX. As NaN values can cause unexpected behaviour with + called .nosatflags_wlX. As NaN values can cause unexpected behaviour with mathematical functions the default behaviour is to return the saturated data. However, you may request the data with saturated segments replaced with NaN by setting the saturated argument to nan. diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 83aebc57ab6..492defcb679 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -32,6 +32,52 @@ 'NIRx', 'nirscout', 'nirx_15_3_recording') +nirsport1_wo_sat = op.join(data_path(download=False), + 'NIRx', 'nirsport_v1', + 'nirx_15_3_recording_wo_saturation') +nirsport1_w_sat = op.join(data_path(download=False), + 'NIRx', 'nirsport_v1', + 'nirx_15_3_recording_w_occasional_saturation') +nirsport1_w_fullsat = op.join(data_path(download=False), + 'NIRx', 'nirsport_v1', + 'nirx_15_3_recording_w_full_saturation') + + +@requires_testing_data +@pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') +def test_nirsport_v1_wo_sat(): + """Test reading NIRX files using path to header file.""" + raw = read_raw_nirx(nirsport1_wo_sat, preload=True) + + # Test data import + assert raw._data.shape == (26, 164) + assert raw.info['sfreq'] == 10.416667 + + +@pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') +@pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') +@requires_testing_data +def test_nirsport_v1_w_sat(): + """Test reading NIRX files using path to header file.""" + raw = read_raw_nirx(nirsport1_w_sat, preload=True) + + # Test data import + assert raw._data.shape == (26, 176) + assert raw.info['sfreq'] == 10.416667 + + +@pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') +@pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') +@requires_testing_data +def test_nirsport_v1_w_bad_sat(): + """Test reading NIRX files using path to header file.""" + raw = read_raw_nirx(nirsport1_w_fullsat, preload=True) + + # Test data import + assert raw._data.shape == (26, 168) + assert raw.info['sfreq'] == 10.416667 + + @requires_testing_data def test_nirx_hdr_load(): """Test reading NIRX files using path to header file.""" From 51154aaa2b705d16e8c711331722593098e5de96 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 16:37:42 +1000 Subject: [PATCH 09/30] Test that nans are returned. FAILING --- mne/io/nirx/tests/test_nirx.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 492defcb679..575b2f0919e 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -7,6 +7,7 @@ import shutil import os import datetime as dt +import numpy as np import pytest from numpy.testing import assert_allclose, assert_array_equal @@ -32,15 +33,12 @@ 'NIRx', 'nirscout', 'nirx_15_3_recording') -nirsport1_wo_sat = op.join(data_path(download=False), - 'NIRx', 'nirsport_v1', +nirsport1_wo_sat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', 'nirx_15_3_recording_wo_saturation') -nirsport1_w_sat = op.join(data_path(download=False), - 'NIRx', 'nirsport_v1', - 'nirx_15_3_recording_w_occasional_saturation') -nirsport1_w_fullsat = op.join(data_path(download=False), - 'NIRx', 'nirsport_v1', - 'nirx_15_3_recording_w_full_saturation') +nirsport1_w_sat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', + 'nirx_15_3_recording_w_occasional_saturation') +nirsport1_w_fullsat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', + 'nirx_15_3_recording_w_full_saturation') @requires_testing_data @@ -53,6 +51,9 @@ def test_nirsport_v1_wo_sat(): assert raw._data.shape == (26, 164) assert raw.info['sfreq'] == 10.416667 + # By default real data is returned + assert np.sum(np.isnan(raw._data)) == 0 + @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @@ -65,6 +66,14 @@ def test_nirsport_v1_w_sat(): assert raw._data.shape == (26, 176) assert raw.info['sfreq'] == 10.416667 + # By default real data is returned + assert np.sum(np.isnan(raw._data)) == 0 + + raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') + assert np.isnan(raw._data).any() == False + # I am confused, why doesnt the test above have nans + # By my understanding the data should contain NaNs + @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @@ -77,6 +86,14 @@ def test_nirsport_v1_w_bad_sat(): assert raw._data.shape == (26, 168) assert raw.info['sfreq'] == 10.416667 + # By default real data is returned + assert np.sum(np.isnan(raw._data)) == 0 + + raw = read_raw_nirx(nirsport1_w_fullsat, preload=True, saturated='nan') + assert np.isnan(raw._data).any() == False + # I am confused, why doesnt the test above have nans + # By my understanding the data should contain NaNs + @requires_testing_data def test_nirx_hdr_load(): From 911e6f4d505e6009dfbe6bc474fc289f6ec8d452 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 16:44:11 +1000 Subject: [PATCH 10/30] More tests. Still strange behaviour --- mne/io/nirx/tests/test_nirx.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 575b2f0919e..8d7ae640184 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -69,11 +69,19 @@ def test_nirsport_v1_w_sat(): # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 + # This should return data with NaNs raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') - assert np.isnan(raw._data).any() == False + # assert np.isnan(raw._data).any() is True # I am confused, why doesnt the test above have nans # By my understanding the data should contain NaNs + # The data should contain nans, so the annotation shouldn't throw warning + # Something weird is happening, the data doesnt seem to contain nans. + # raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='annotate') + # assert np.sum(np.isnan(raw._data)) == 0 + # I am confused, why doesnt the data above have nans + # By my understanding the data should contain NaNs + @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') From 2a78edf53a725397f0f8a8496b1aefd0b0c93e01 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 17:09:43 +1000 Subject: [PATCH 11/30] Found cause of strange behaviour --- mne/io/nirx/tests/test_nirx.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 8d7ae640184..02e1f9d4107 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -69,18 +69,16 @@ def test_nirsport_v1_w_sat(): # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 - # This should return data with NaNs - raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') + # Ideally the following function should return NaNs. + # However, the measured data does not have NaNs + # in the channels of interest. + raw2 = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') # assert np.isnan(raw._data).any() is True - # I am confused, why doesnt the test above have nans - # By my understanding the data should contain NaNs # The data should contain nans, so the annotation shouldn't throw warning - # Something weird is happening, the data doesnt seem to contain nans. + # However, the test file does not have nans in the specified channels. # raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='annotate') # assert np.sum(np.isnan(raw._data)) == 0 - # I am confused, why doesnt the data above have nans - # By my understanding the data should contain NaNs @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @@ -97,10 +95,11 @@ def test_nirsport_v1_w_bad_sat(): # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 + # Ideally the following function should return NaNs. + # However, the measured data does not have NaNs + # in the channels of interest. raw = read_raw_nirx(nirsport1_w_fullsat, preload=True, saturated='nan') - assert np.isnan(raw._data).any() == False - # I am confused, why doesnt the test above have nans - # By my understanding the data should contain NaNs + assert np.isnan(raw._data).any() is True @requires_testing_data From a66df0c4c1accb3f37252560340e098a774700a4 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 17:33:51 +1000 Subject: [PATCH 12/30] Flake --- mne/io/nirx/tests/test_nirx.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 02e1f9d4107..316308a827d 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -72,7 +72,8 @@ def test_nirsport_v1_w_sat(): # Ideally the following function should return NaNs. # However, the measured data does not have NaNs # in the channels of interest. - raw2 = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') + raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') + assert raw._data.shape == (26, 176) # assert np.isnan(raw._data).any() is True # The data should contain nans, so the annotation shouldn't throw warning @@ -99,7 +100,8 @@ def test_nirsport_v1_w_bad_sat(): # However, the measured data does not have NaNs # in the channels of interest. raw = read_raw_nirx(nirsport1_w_fullsat, preload=True, saturated='nan') - assert np.isnan(raw._data).any() is True + assert raw._data.shape == (26, 168) + # assert np.isnan(raw._data).any() is True @requires_testing_data From 5b23b87f0f350ead5de31fad20373ad9028a1e67 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Sat, 24 Apr 2021 17:52:13 +1000 Subject: [PATCH 13/30] Add annotation description to docs --- mne/io/nirx/nirx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 4d899dc3a9e..0cd458c648e 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -34,6 +34,8 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): contains measurements while the amplifier was saturated. If nan the returned data will contain NaNs during time segments when the amplifier was saturated. + If annotate the returned data will contain annotations specifying + sections the saturate segments. This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). %(preload)s @@ -82,6 +84,8 @@ class RawNIRX(BaseRaw): contains measurements while the amplifier was saturated. If nan the returned data will contain NaNs during time segments when the amplifier was saturated. + If annotate the returned data will contain annotations specifying + sections the saturate segments. This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). From 097df9418b3b31b0ded4e78add6295fe88832bce Mon Sep 17 00:00:00 2001 From: Robert Luke <748691+rob-luke@users.noreply.github.com> Date: Wed, 28 Apr 2021 14:53:10 +1000 Subject: [PATCH 14/30] Apply suggestions from code review Co-authored-by: Eric Larson --- mne/io/nirx/nirx.py | 6 +++--- mne/preprocessing/annotate_nan.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 0cd458c648e..be39477f68d 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -30,11 +30,11 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): Path to the NIRX data folder or header file. saturated : str Replace saturated segments of data with NaNs. - If ignore (default) the measured data is returned, even if it + If "ignore" (default) the measured data is returned, even if it contains measurements while the amplifier was saturated. - If nan the returned data will contain NaNs during time segments + If "nan" the returned data will contain NaNs during time segments when the amplifier was saturated. - If annotate the returned data will contain annotations specifying + If "annotate" the returned data will contain annotations specifying sections the saturate segments. This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index 8bfb3d2ef47..3be694b9ead 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -10,7 +10,8 @@ from ..utils import _mask_to_onsets_offsets -def annotate_nan(raw): +@verbose +def annotate_nan(raw, *, verbose=None): """Detect segments with NaN and return a new Annotations instance. Parameters From 75ea78d78b9d8d60130c9f5853464ee7081faac4 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 28 Apr 2021 14:53:59 +1000 Subject: [PATCH 15/30] Spelling --- mne/preprocessing/annotate_nan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index 8bfb3d2ef47..50e75df70d7 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -36,7 +36,7 @@ def annotate_nan(raw): durations = (stops - starts) * sampling_duration else: warnings.warn("The dataset you provided does not contain 'NaN' " - "values. No annotation were made.") + "values. No annotations were made.") return if annot is None: From 278c04b74f533a1ada0debb114f1f5fc8c5396e4 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 28 Apr 2021 14:56:39 +1000 Subject: [PATCH 16/30] Fix verbose --- mne/preprocessing/annotate_nan.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index 32a6e089d1c..73f7a7e6ed2 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -7,17 +7,18 @@ import warnings from ..annotations import Annotations -from ..utils import _mask_to_onsets_offsets +from ..utils import _mask_to_onsets_offsets, verbose @verbose -def annotate_nan(raw, *, verbose=None): +def annotate_nan(raw, verbose=None): """Detect segments with NaN and return a new Annotations instance. Parameters ---------- raw : instance of Raw Data to find segments with NaN values. + %(verbose)s Returns ------- From 8c6a8e13bdcf1fffe6573aa96880243ca2b3f38f Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 28 Apr 2021 15:01:03 +1000 Subject: [PATCH 17/30] Further suggestions from review --- mne/preprocessing/annotate_nan.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index 73f7a7e6ed2..d841168d2f4 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -7,7 +7,7 @@ import warnings from ..annotations import Annotations -from ..utils import _mask_to_onsets_offsets, verbose +from ..utils import _mask_to_onsets_offsets, verbose, warn @verbose @@ -32,18 +32,15 @@ def annotate_nan(raw, verbose=None): nans = np.any(np.isnan(data), axis=0) starts, stops = _mask_to_onsets_offsets(nans) - if len(starts) > 0: - starts, stops = np.array(starts), np.array(stops) - onsets = (starts + raw.first_samp) * sampling_duration - durations = (stops - starts) * sampling_duration - else: - warnings.warn("The dataset you provided does not contain 'NaN' " - "values. No annotations were made.") - return - - if annot is None: - annot = Annotations(onsets, durations, 'bad_NAN') - else: - annot.append(onsets, durations, 'bad_NAN') + if len(starts) == 0: + warn("The dataset you provided does not contain 'NaN' values. " + "No annotations were made.") + return annot + + starts, stops = np.array(starts), np.array(stops) + onsets = (starts + raw.first_samp) * sampling_duration + durations = (stops - starts) * sampling_duration + + annot.append(onsets, durations, 'bad_NAN') return annot From db59e1a005eda522e0da358b031cc89bda6f5de5 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 28 Apr 2021 15:04:40 +1000 Subject: [PATCH 18/30] Use new test file --- mne/io/nirx/tests/test_nirx.py | 50 ++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 316308a827d..8a6ba2e1be2 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -33,18 +33,21 @@ 'NIRx', 'nirscout', 'nirx_15_3_recording') +# This file has no saturated sections nirsport1_wo_sat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', 'nirx_15_3_recording_wo_saturation') +# This file has saturation, but not on the optode pairing in montage nirsport1_w_sat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', 'nirx_15_3_recording_w_occasional_saturation') +# This file has saturation in channels of interest nirsport1_w_fullsat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', - 'nirx_15_3_recording_w_full_saturation') + 'ECST') @requires_testing_data @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') def test_nirsport_v1_wo_sat(): - """Test reading NIRX files using path to header file.""" + """Test NIRSport1 file with no saturation.""" raw = read_raw_nirx(nirsport1_wo_sat, preload=True) # Test data import @@ -54,32 +57,36 @@ def test_nirsport_v1_wo_sat(): # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 + raw = read_raw_nirx(nirsport1_wo_sat, preload=True, saturated='nan') + assert raw._data.shape == (26, 164) + assert np.sum(np.isnan(raw._data)) == 0 + + with pytest.raises(RuntimeWarning, match='does not contain'): + raw = read_raw_nirx(nirsport1_wo_sat, saturated='annotate') + assert raw._data.shape == (26, 164) + assert np.sum(np.isnan(raw._data)) == 0 + @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @requires_testing_data def test_nirsport_v1_w_sat(): - """Test reading NIRX files using path to header file.""" + """Test NIRSport1 file with NaNs but not in channel of interest.""" raw = read_raw_nirx(nirsport1_w_sat, preload=True) # Test data import assert raw._data.shape == (26, 176) assert raw.info['sfreq'] == 10.416667 - - # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 - # Ideally the following function should return NaNs. - # However, the measured data does not have NaNs - # in the channels of interest. raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') assert raw._data.shape == (26, 176) - # assert np.isnan(raw._data).any() is True + assert np.sum(np.isnan(raw._data)) == 0 - # The data should contain nans, so the annotation shouldn't throw warning - # However, the test file does not have nans in the specified channels. - # raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='annotate') - # assert np.sum(np.isnan(raw._data)) == 0 + with pytest.raises(RuntimeWarning, match='does not contain'): + raw = read_raw_nirx(nirsport1_w_sat, saturated='annotate') + assert raw._data.shape == (26, 176) + assert np.sum(np.isnan(raw._data)) == 0 @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @@ -90,18 +97,21 @@ def test_nirsport_v1_w_bad_sat(): raw = read_raw_nirx(nirsport1_w_fullsat, preload=True) # Test data import - assert raw._data.shape == (26, 168) - assert raw.info['sfreq'] == 10.416667 + assert raw._data.shape == (56, 2339) + assert raw.info['sfreq'] == 7.8125 # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 - # Ideally the following function should return NaNs. - # However, the measured data does not have NaNs - # in the channels of interest. raw = read_raw_nirx(nirsport1_w_fullsat, preload=True, saturated='nan') - assert raw._data.shape == (26, 168) - # assert np.isnan(raw._data).any() is True + assert raw._data.shape == (56, 2339) + assert np.sum(np.isnan(raw._data)) > 1 + assert 'bad_NAN' not in raw.annotations.description + + raw = read_raw_nirx(nirsport1_w_fullsat, saturated='annotate') + assert raw.load_data()._data.shape == (56, 2339) + assert np.sum(np.isnan(raw._data)) > 1 + assert 'bad_NAN' in raw.annotations.description @requires_testing_data From 79e3b37bdfe1634b9b323be6b9faa9493ec4d432 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 28 Apr 2021 15:14:58 +1000 Subject: [PATCH 19/30] Flake --- mne/preprocessing/annotate_nan.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index d841168d2f4..3b69bfab675 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -4,9 +4,6 @@ import numpy as np -import warnings - -from ..annotations import Annotations from ..utils import _mask_to_onsets_offsets, verbose, warn From ad8e112c9c0891dc8a1c27935658498f5f34ea39 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Thu, 29 Apr 2021 19:58:21 +1000 Subject: [PATCH 20/30] Fix bug in logic --- mne/io/nirx/nirx.py | 16 ++++++++++++++-- mne/io/nirx/tests/test_nirx.py | 14 ++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index be39477f68d..abc872c11ec 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -133,15 +133,27 @@ def __init__(self, fname, saturated, preload=False, verbose=None): if (key == 'wl1' or key == 'wl2') \ and len(glob.glob('%s/*%s' % (fname, 'nosatflags_' + key))) == 1: + + # Here two files have been found, one that is called + # no sat flags. The nosatflag file has no NaNs in it. + # The wlX file has NaNs in it. + # First we determine which of the files has the saturation + # flags in it. + satidx = np.where(['nosatflags' not in op.basename(k) + for k in files[key]])[0][0] + if saturated == 'nan': + # In this case data is returned with NaNs in it. warn('The measurement contains saturated data. ' 'Saturated values will be replaced by NaNs.') - files[key] = files[key][1] + files[key] = files[key][satidx] elif saturated == 'annotate': + # In this case data is returned with NaNs in it and + # annotations will be made with `bad_NAN` warn('The measurement contains saturated data. ' 'Saturated values will be annotated ' 'with \'nan\' flags.') - files[key] = files[key][1] + files[key] = files[key][satidx] else: if saturated == 'ignore': warn('The measurement contains saturated data.') diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 8a6ba2e1be2..f72b92fcddd 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -38,10 +38,10 @@ 'nirx_15_3_recording_wo_saturation') # This file has saturation, but not on the optode pairing in montage nirsport1_w_sat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', - 'nirx_15_3_recording_w_occasional_saturation') + 'nirx_15_3_recording_w_saturation_not_on_montage_channels') # This file has saturation in channels of interest nirsport1_w_fullsat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', - 'ECST') + 'nirx_15_3_recording_w_saturation_on_montage_channels') @requires_testing_data @@ -96,20 +96,18 @@ def test_nirsport_v1_w_bad_sat(): """Test reading NIRX files using path to header file.""" raw = read_raw_nirx(nirsport1_w_fullsat, preload=True) - # Test data import - assert raw._data.shape == (56, 2339) - assert raw.info['sfreq'] == 7.8125 - # By default real data is returned assert np.sum(np.isnan(raw._data)) == 0 + assert raw._data.shape == (26, 168) raw = read_raw_nirx(nirsport1_w_fullsat, preload=True, saturated='nan') - assert raw._data.shape == (56, 2339) assert np.sum(np.isnan(raw._data)) > 1 + assert raw._data.shape == (26, 168) assert 'bad_NAN' not in raw.annotations.description raw = read_raw_nirx(nirsport1_w_fullsat, saturated='annotate') - assert raw.load_data()._data.shape == (56, 2339) + raw.load_data() + assert raw._data.shape == (26, 168) assert np.sum(np.isnan(raw._data)) > 1 assert 'bad_NAN' in raw.annotations.description From b7544ff90a7dc7aee0e4886a06325c3bdb2db23c Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Thu, 29 Apr 2021 20:08:00 +1000 Subject: [PATCH 21/30] Flake --- mne/io/nirx/tests/test_nirx.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index f72b92fcddd..fa19e22eaba 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -38,10 +38,12 @@ 'nirx_15_3_recording_wo_saturation') # This file has saturation, but not on the optode pairing in montage nirsport1_w_sat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', - 'nirx_15_3_recording_w_saturation_not_on_montage_channels') + 'nirx_15_3_recording_w_saturation_' + 'not_on_montage_channels') # This file has saturation in channels of interest nirsport1_w_fullsat = op.join(data_path(download=False), 'NIRx', 'nirsport_v1', - 'nirx_15_3_recording_w_saturation_on_montage_channels') + 'nirx_15_3_recording_w_' + 'saturation_on_montage_channels') @requires_testing_data From 2f70d8a92aff6f79f3fe9a6257eac51277da6fcc Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Thu, 29 Apr 2021 20:09:24 +1000 Subject: [PATCH 22/30] Doc --- mne/io/nirx/tests/test_nirx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index fa19e22eaba..29c7800696f 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -95,7 +95,7 @@ def test_nirsport_v1_w_sat(): @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @requires_testing_data def test_nirsport_v1_w_bad_sat(): - """Test reading NIRX files using path to header file.""" + """Test NIRSport1 file with NaNs.""" raw = read_raw_nirx(nirsport1_w_fullsat, preload=True) # By default real data is returned From 7fe2fd022c7e773f75255a0f4ddfc5be5f07c676 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Thu, 29 Apr 2021 20:14:59 +1000 Subject: [PATCH 23/30] doc --- mne/io/nirx/nirx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index abc872c11ec..a8d60d78c69 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -35,7 +35,7 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): If "nan" the returned data will contain NaNs during time segments when the amplifier was saturated. If "annotate" the returned data will contain annotations specifying - sections the saturate segments. + sections the saturate segments and the data will contain NaNs. This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). %(preload)s @@ -85,7 +85,7 @@ class RawNIRX(BaseRaw): If nan the returned data will contain NaNs during time segments when the amplifier was saturated. If annotate the returned data will contain annotations specifying - sections the saturate segments. + sections the saturate segments and the data will contain NaNs. This argument will only be used if there is no .nosatflags file (only if a NIRSport device is used and saturation occurred). From fbb47432d44a3eb630bc3f48f5a861f31c6d7c81 Mon Sep 17 00:00:00 2001 From: Robert Luke <748691+rob-luke@users.noreply.github.com> Date: Thu, 29 Apr 2021 21:53:11 +1000 Subject: [PATCH 24/30] Update utils.py --- mne/datasets/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/datasets/utils.py b/mne/datasets/utils.py index 6df444191d2..9b824fbe589 100644 --- a/mne/datasets/utils.py +++ b/mne/datasets/utils.py @@ -254,7 +254,7 @@ def _data_path(path=None, force_update=False, update_path=True, download=True, path = _get_path(path, key, name) # To update the testing or misc dataset, push commits, then make a new # release on GitHub. Then update the "releases" variable: - releases = dict(testing='0.117', misc='0.9') + releases = dict(testing='0.119', misc='0.9') # And also update the "md5_hashes['testing']" variable below. # To update any other dataset, update the data archive itself (upload # an updated version) and update the md5 hash. @@ -349,7 +349,7 @@ def _data_path(path=None, force_update=False, update_path=True, download=True, sample='12b75d1cb7df9dfb4ad73ed82f61094f', somato='32fd2f6c8c7eb0784a1de6435273c48b', spm='9f43f67150e3b694b523a21eb929ea75', - testing='d8df35b2e625e213769e97e719de205c', + testing='2e7c60a055228928bd39f68892b3d488', multimodal='26ec847ae9ab80f58f204d09e2c08367', fnirs_motor='c4935d19ddab35422a69f3326a01fef8', opm='370ad1dcfd5c47e029e692c85358a374', From 5f76cdf37af180bacaf77c3e67dff84b9c85c114 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 3 May 2021 13:56:11 -0400 Subject: [PATCH 25/30] FIX: Two-pass approach if needed --- mne/io/nirx/nirx.py | 187 +++++++++++++----------------- mne/io/nirx/tests/test_nirx.py | 60 ++++------ mne/preprocessing/annotate_nan.py | 22 +--- mne/utils/docs.py | 30 +++++ 4 files changed, 138 insertions(+), 161 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index a8d60d78c69..33c5f26faeb 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -17,27 +17,18 @@ from ...annotations import Annotations from ...transforms import apply_trans, _get_trans from ...utils import (logger, verbose, fill_doc, warn, _check_fname, - _validate_type) + _validate_type, _check_option, _mask_to_onsets_offsets) @fill_doc -def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): +def read_raw_nirx(fname, saturated='annotate', preload=False, verbose=None): """Reader for a NIRX fNIRS recording. Parameters ---------- fname : str Path to the NIRX data folder or header file. - saturated : str - Replace saturated segments of data with NaNs. - If "ignore" (default) the measured data is returned, even if it - contains measurements while the amplifier was saturated. - If "nan" the returned data will contain NaNs during time segments - when the amplifier was saturated. - If "annotate" the returned data will contain annotations specifying - sections the saturate segments and the data will contain NaNs. - This argument will only be used if there is no .nosatflags file - (only if a NIRSport device is used and saturation occurred). + %(saturated)s %(preload)s %(verbose)s @@ -52,16 +43,7 @@ def read_raw_nirx(fname, saturated='ignore', preload=False, verbose=None): Notes ----- - This function has only been tested with NIRScout and NIRSport1 devices. - - The NIRSport device can detect if the amplifier is saturated. - Starting from NIRStar 14.2, those saturated values are replaced by NaNs - in the standard .wlX files. - The raw unmodified measured values are stored in another file - called .nosatflags_wlX. As NaN values can cause unexpected behaviour with - mathematical functions the default behaviour is to return the - saturated data. However, you may request the data with saturated - segments replaced with NaN by setting the saturated argument to nan. + %(nirx_notes)s """ return RawNIRX(fname, saturated, preload, verbose) @@ -78,17 +60,7 @@ class RawNIRX(BaseRaw): ---------- fname : str Path to the NIRX data folder or header file. - saturated : str - Replace saturated segments of data with NaNs. - If ignore (default) the measured data is returned, even if it - contains measurements while the amplifier was saturated. - If nan the returned data will contain NaNs during time segments - when the amplifier was saturated. - If annotate the returned data will contain annotations specifying - sections the saturate segments and the data will contain NaNs. - This argument will only be used if there is no .nosatflags file - (only if a NIRSport device is used and saturation occurred). - + %(saturated)s %(preload)s %(verbose)s @@ -98,25 +70,17 @@ class RawNIRX(BaseRaw): Notes ----- - This function has only been tested with NIRScout and NIRSport1 devices. - - The NIRSport device can detect if the amplifier is saturated. - Starting from NIRStar 14.2, those saturated values are replaced by NaNs - in the standard .wlX files. - The raw unmodified measured values are stored in another file - called .nosatflags_wlX. As NaN values can cause unexpected behaviour with - mathematical functions the default behaviour is to return the - saturated data. However, you may request the data with saturated - segments replaced with NaN by setting the saturated argument to nan. + %(nirx_notes)s """ @verbose def __init__(self, fname, saturated, preload=False, verbose=None): from ...externals.pymatreader import read_mat from ...coreg import get_mni_fiducials # avoid circular import prob - from ...preprocessing import annotate_nan # avoid circular import prob logger.info('Loading %s' % fname) _validate_type(fname, 'path-like', 'fname') + _validate_type(saturated, str, 'saturated') + _check_option('saturated', saturated, ('annotate', 'nan', 'ignore')) fname = str(fname) if fname.endswith('.hdr'): fname = op.dirname(op.abspath(fname)) @@ -127,49 +91,34 @@ def __init__(self, fname, saturated, preload=False, verbose=None): files = dict() keys = ('hdr', 'inf', 'set', 'tpl', 'wl1', 'wl2', 'config.txt', 'probeInfo.mat') + nan_mask = dict() for key in keys: files[key] = glob.glob('%s/*%s' % (fname, key)) + fidx = 0 if len(files[key]) != 1: - if (key == 'wl1' or key == 'wl2') \ - and len(glob.glob('%s/*%s' % - (fname, 'nosatflags_' + key))) == 1: - - # Here two files have been found, one that is called - # no sat flags. The nosatflag file has no NaNs in it. - # The wlX file has NaNs in it. - # First we determine which of the files has the saturation - # flags in it. - satidx = np.where(['nosatflags' not in op.basename(k) - for k in files[key]])[0][0] - - if saturated == 'nan': - # In this case data is returned with NaNs in it. - warn('The measurement contains saturated data. ' - 'Saturated values will be replaced by NaNs.') - files[key] = files[key][satidx] - elif saturated == 'annotate': - # In this case data is returned with NaNs in it and - # annotations will be made with `bad_NAN` - warn('The measurement contains saturated data. ' - 'Saturated values will be annotated ' - 'with \'nan\' flags.') - files[key] = files[key][satidx] - else: - if saturated == 'ignore': - warn('The measurement contains saturated data.') - else: - raise KeyError("The value specified for saturated " - "must be one of ignore, nan, or " - "annotate. " - f"{saturated} was provided") - - files[key] = glob.glob('%s/*%s' % - (fname, 'nosatflags_' + key))[0] + if key not in ('wl1', 'wl2'): + raise RuntimeError( + f'Need one {key} file, got {len(files[key])}') + noidx = np.where(['nosatflags_' in op.basename(x) + for x in files[key]])[0] + if len(noidx) != 1 or len(files[key]) != 2: + raise RuntimeError( + f'Need one nosatflags and one standard {key} file, ' + f'got {len(files[key])}') + # Here two files have been found, one that is called + # no sat flags. The nosatflag file has no NaNs in it. + noidx = noidx[0] + if saturated == 'ignore': + # Ignore NaN and return values + fidx = noidx + elif saturated == 'nan': + # Return NaN + fidx = 0 if noidx == 1 else 1 else: - raise RuntimeError('Expect one %s file, got %d' % - (key, len(files[key]),)) - else: - files[key] = files[key][0] + assert saturated == 'annotate' # guaranteed above + fidx = noidx + nan_mask[key] = files[key][0 if noidx == 1 else 1] + files[key] = files[key][fidx] if len(glob.glob('%s/*%s' % (fname, 'dat'))) != 1: warn("A single dat file was expected in the specified path, but " "got %d. This may indicate that the file structure has been " @@ -177,10 +126,8 @@ def __init__(self, fname, saturated, preload=False, verbose=None): (len(glob.glob('%s/*%s' % (fname, 'dat'))))) # Read number of rows/samples of wavelength data - last_sample = -1 with _open(files['wl1']) as fid: - for line in fid: - last_sample += 1 + last_sample = fid.read().count('\n') - 1 # Read header file # The header file isn't compliant with the configparser. So all the @@ -383,31 +330,49 @@ def prepend(li, str): 'sd_index': req_ind, 'files': files, 'bounds': bounds, + 'nan_mask': nan_mask, } + # Get our saturated mask + annot_mask = None + for key in ('wl1', 'wl2'): + if nan_mask.get(key, None) is None: + continue + nan_mask[key] = np.isnan(_read_csv_rows_cols( + nan_mask[key], 0, last_sample + 1, req_ind, {0: 0, 1: None}).T) + if saturated == 'annotate': + if annot_mask is None: + annot_mask = nan_mask[key] + else: + annot_mask |= nan_mask[key] + nan_mask[key] = None # shouldn't need again super(RawNIRX, self).__init__( info, preload, filenames=[fname], last_samps=[last_sample], raw_extras=[raw_extras], verbose=verbose) + # make onset/duration/description + onset, duration, description = list(), list(), list() + if annot_mask is not None: + on, dur = _mask_to_onsets_offsets(annot_mask.any(0)) + on = on / info['sfreq'] + dur = dur / info['sfreq'] + dur -= on + onset.extend(on) + duration.extend(dur) + description.extend(['BAD_NAN'] * len(on)) + # Read triggers from event file if op.isfile(files['hdr'][:-3] + 'evt'): with _open(files['hdr'][:-3] + 'evt') as fid: t = [re.findall(r'(\d+)', line) for line in fid] - onset = np.zeros(len(t), float) - duration = np.zeros(len(t), float) - description = [''] * len(t) - for t_idx in range(len(t)): - binary_value = ''.join(t[t_idx][1:])[::-1] - trigger_frame = float(t[t_idx][0]) - onset[t_idx] = (trigger_frame) * (1.0 / samplingrate) - duration[t_idx] = 1.0 # No duration info stored in files - description[t_idx] = int(binary_value, 2) * 1. - annot = Annotations(onset, duration, description) - self.set_annotations(annot) - - if saturated == "annotate": - annot = annotate_nan(self) - self.set_annotations(annot) + for t_ in t: + binary_value = ''.join(t_[1:])[::-1] + trigger_frame = float(t_[0]) + onset.append(trigger_frame / samplingrate) + duration.append(1.) # No duration info stored in files + description.append(float(int(binary_value, 2))) + annot = Annotations(onset, duration, description) + self.set_annotations(annot) def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): """Read a segment of data from a file. @@ -415,15 +380,18 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): The NIRX machine records raw data as two different wavelengths. The returned data interleaves the wavelengths. """ - sdindex = self._raw_extras[fi]['sd_index'] + sd_index = self._raw_extras[fi]['sd_index'] - wls = [ - _read_csv_rows_cols( + wls = list() + for key in ('wl1', 'wl2'): + d = _read_csv_rows_cols( self._raw_extras[fi]['files'][key], - start, stop, sdindex, + start, stop, sd_index, self._raw_extras[fi]['bounds'][key]).T - for key in ('wl1', 'wl2') - ] + nan_mask = self._raw_extras[fi]['nan_mask'].get(key, None) + if nan_mask is not None: + d[nan_mask[:, start:stop]] = np.nan + wls.append(d) # TODO: Make this more efficient by only indexing above what we need. # For now let's just construct the full data matrix and index. @@ -438,7 +406,10 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): def _read_csv_rows_cols(fname, start, stop, cols, bounds): with open(fname, 'rb') as fid: fid.seek(bounds[start]) - data = fid.read(bounds[stop] - bounds[start]).decode('latin-1') + args = list() + if bounds[1] is not None: + args.append(bounds[stop] - bounds[start]) + data = fid.read(*args).decode('latin-1') x = np.fromstring(data, float, sep=' ') x.shape = (stop - start, -1) x = x[:, cols] diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index f39fe963d42..285e838c4ea 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -16,6 +16,7 @@ from mne.datasets.testing import data_path, requires_testing_data from mne.io import read_raw_nirx from mne.io.tests.test_raw import _test_raw_reader +from mne.preprocessing import annotate_nan from mne.transforms import apply_trans, _get_trans from mne.preprocessing.nirs import source_detector_distances,\ short_channels @@ -93,24 +94,31 @@ def test_nirsport_v1_w_sat(): @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @requires_testing_data -def test_nirsport_v1_w_bad_sat(): +@pytest.mark.parametrize('preload', (True, False)) +def test_nirsport_v1_w_bad_sat(preload): """Test NIRSport1 file with NaNs.""" - raw = read_raw_nirx(nirsport1_w_fullsat, preload=True) - - # By default real data is returned - assert np.sum(np.isnan(raw._data)) == 0 - assert raw._data.shape == (26, 168) - - raw = read_raw_nirx(nirsport1_w_fullsat, preload=True, saturated='nan') - assert np.sum(np.isnan(raw._data)) > 1 - assert raw._data.shape == (26, 168) - assert 'bad_NAN' not in raw.annotations.description - - raw = read_raw_nirx(nirsport1_w_fullsat, saturated='annotate') - raw.load_data() - assert raw._data.shape == (26, 168) - assert np.sum(np.isnan(raw._data)) > 1 - assert 'bad_NAN' in raw.annotations.description + fname = nirsport1_w_fullsat + raw = read_raw_nirx(fname, preload=preload) + data = raw.get_data() + assert not np.isnan(data).any() + assert len(raw.annotations) == 5 + # annotated version and ignore should have same data but different annot + raw_ignore = read_raw_nirx(fname, saturated='ignore', preload=preload) + assert_allclose(raw_ignore.get_data(), data) + assert len(raw_ignore.annotations) == 2 + assert not any('NAN' in d for d in raw_ignore.annotations.description) + # nan version should not have same data, but we can give it the same annot + raw_nan = read_raw_nirx(fname, saturated='nan', preload=preload) + data_nan = raw_nan.get_data() + assert np.isnan(data_nan).any() # XXX should be a better accounting + assert not np.allclose(raw_nan.get_data(), data) + raw_nan_annot = raw_ignore.copy() + raw_nan_annot.set_annotations(annotate_nan(raw_nan)) + use_mask = np.where(raw.annotations.description == 'BAD_NAN') + for key in ('onset', 'duration'): + a = getattr(raw_nan_annot.annotations, key) + b = getattr(raw.annotations, key)[use_mask] + assert_allclose(a, b) @requires_testing_data @@ -153,24 +161,6 @@ def test_nirx_dat_warn(tmpdir): read_raw_nirx(fname, preload=True) -@requires_testing_data -def test_nirx_nosatflags_v1_warn(tmpdir): - """Test reading NIRSportv1 files with saturated data.""" - shutil.copytree(fname_nirx_15_2_short, str(tmpdir) + "/data/") - shutil.copyfile(str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.wl1", - str(tmpdir) + "/data" + - "/NIRS-2019-08-23_001.nosatflags_wl1") - fname = str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.hdr" - with pytest.raises(RuntimeWarning, match='be replaced by NaNs'): - read_raw_nirx(fname, saturated='nan', preload=True) - with pytest.raises(RuntimeWarning, match='annotated with \'nan\' flags'): - read_raw_nirx(fname, saturated='annotate', preload=True) - with pytest.raises(RuntimeWarning, match='contains saturated data'): - read_raw_nirx(fname, saturated='ignore', preload=True) - with pytest.raises(KeyError, match='specified for saturated must'): - read_raw_nirx(fname, saturated='foobar', preload=True) - - @requires_testing_data def test_nirx_15_2_short(): """Test reading NIRX files.""" diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index 3b69bfab675..8028ccba7b0 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -4,11 +4,12 @@ import numpy as np -from ..utils import _mask_to_onsets_offsets, verbose, warn +from ..utils import verbose +from .artifact_detection import _annotations_from_mask @verbose -def annotate_nan(raw, verbose=None): +def annotate_nan(raw, *, verbose=None): """Detect segments with NaN and return a new Annotations instance. Parameters @@ -22,22 +23,7 @@ def annotate_nan(raw, verbose=None): annot : instance of Annotations Updated annotations for raw data. """ - annot = raw.annotations.copy() data, times = raw.get_data(return_times=True) - sampling_duration = 1 / raw.info['sfreq'] - nans = np.any(np.isnan(data), axis=0) - starts, stops = _mask_to_onsets_offsets(nans) - - if len(starts) == 0: - warn("The dataset you provided does not contain 'NaN' values. " - "No annotations were made.") - return annot - - starts, stops = np.array(starts), np.array(stops) - onsets = (starts + raw.first_samp) * sampling_duration - durations = (stops - starts) * sampling_duration - - annot.append(onsets, durations, 'bad_NAN') - + annot = _annotations_from_mask(times, nans, 'BAD_NAN') return annot diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 435fcc3149e..3e38b8276ea 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -58,6 +58,36 @@ .. versionadded:: 0.22 """ % (_on_missing_base,) +docdict['saturated'] = """\ +saturated : str + Replace saturated segments of data with NaNs, can be: + + ``"ignore"`` + The measured data is returned, even if it contains measurements + while the amplifier was saturated. + ``"nan"`` + The returned data will contain NaNs during time segments + when the amplifier was saturated. + ``"annotate"`` (default) + The returned data will contain annotations specifying + sections the saturate segments. + + This argument will only be used if there is no .nosatflags file + (only if a NIRSport device is used and saturation occurred). + + .. versionadded:: 0.24 +""" +docdict['nirx_notes'] = """\ +This function has only been tested with NIRScout and NIRSport1 devices. + +The NIRSport device can detect if the amplifier is saturated. +Starting from NIRStar 14.2, those saturated values are replaced by NaNs +in the standard .wlX files. +The raw unmodified measured values are stored in another file +called .nosatflags_wlX. As NaN values can cause unexpected behaviour with +mathematical functions the default behaviour is to return the +saturated data. +""" # Cropping docdict['include_tmax'] = """ From 950779067efe0e09ec88474a494cdda6dde56e20 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 3 May 2021 13:59:05 -0400 Subject: [PATCH 26/30] FIX: Missed some --- mne/io/nirx/tests/test_nirx.py | 37 ++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 285e838c4ea..7bbcee01b65 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -57,16 +57,17 @@ def test_nirsport_v1_wo_sat(): assert raw.info['sfreq'] == 10.416667 # By default real data is returned - assert np.sum(np.isnan(raw._data)) == 0 + assert np.sum(np.isnan(raw.get_data())) == 0 raw = read_raw_nirx(nirsport1_wo_sat, preload=True, saturated='nan') - assert raw._data.shape == (26, 164) - assert np.sum(np.isnan(raw._data)) == 0 + data = raw.get_data() + assert data.shape == (26, 164) + assert np.sum(np.isnan(data)) == 0 - with pytest.raises(RuntimeWarning, match='does not contain'): - raw = read_raw_nirx(nirsport1_wo_sat, saturated='annotate') - assert raw._data.shape == (26, 164) - assert np.sum(np.isnan(raw._data)) == 0 + raw = read_raw_nirx(nirsport1_wo_sat, saturated='annotate') + data = raw.get_data() + assert data.shape == (26, 164) + assert np.sum(np.isnan(data)) == 0 @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @@ -74,21 +75,23 @@ def test_nirsport_v1_wo_sat(): @requires_testing_data def test_nirsport_v1_w_sat(): """Test NIRSport1 file with NaNs but not in channel of interest.""" - raw = read_raw_nirx(nirsport1_w_sat, preload=True) + raw = read_raw_nirx(nirsport1_w_sat) # Test data import - assert raw._data.shape == (26, 176) + data = raw.get_data() + assert data.shape == (26, 176) assert raw.info['sfreq'] == 10.416667 - assert np.sum(np.isnan(raw._data)) == 0 + assert np.sum(np.isnan(data)) == 0 - raw = read_raw_nirx(nirsport1_w_sat, preload=True, saturated='nan') - assert raw._data.shape == (26, 176) - assert np.sum(np.isnan(raw._data)) == 0 + raw = read_raw_nirx(nirsport1_w_sat, saturated='nan') + data = raw.get_data() + assert data.shape == (26, 176) + assert np.sum(np.isnan(data)) == 0 - with pytest.raises(RuntimeWarning, match='does not contain'): - raw = read_raw_nirx(nirsport1_w_sat, saturated='annotate') - assert raw._data.shape == (26, 176) - assert np.sum(np.isnan(raw._data)) == 0 + raw = read_raw_nirx(nirsport1_w_sat, saturated='annotate') + data = raw.get_data() + assert data.shape == (26, 176) + assert np.sum(np.isnan(data)) == 0 @pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') From 715042c4d0c5e6eaced09141ede1affa9bda1c1a Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 4 May 2021 08:04:51 -0400 Subject: [PATCH 27/30] ENH: Channel-specific --- mne/io/nirx/nirx.py | 37 ++++++++++++++++++------------- mne/io/nirx/tests/test_nirx.py | 4 ++-- mne/preprocessing/annotate_nan.py | 12 +++++++--- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 33c5f26faeb..1ada7c55ce3 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -334,16 +334,19 @@ def prepend(li, str): } # Get our saturated mask annot_mask = None - for key in ('wl1', 'wl2'): + for ki, key in enumerate(('wl1', 'wl2')): if nan_mask.get(key, None) is None: continue - nan_mask[key] = np.isnan(_read_csv_rows_cols( + mask = np.isnan(_read_csv_rows_cols( nan_mask[key], 0, last_sample + 1, req_ind, {0: 0, 1: None}).T) - if saturated == 'annotate': + if saturated == 'nan': + nan_mask[key] = mask + else: + assert saturated == 'annotate' if annot_mask is None: - annot_mask = nan_mask[key] - else: - annot_mask |= nan_mask[key] + annot_mask = np.zeros( + (len(info['ch_names']), last_sample + 1), bool) + annot_mask[ki::2] = mask nan_mask[key] = None # shouldn't need again super(RawNIRX, self).__init__( @@ -351,15 +354,18 @@ def prepend(li, str): raw_extras=[raw_extras], verbose=verbose) # make onset/duration/description - onset, duration, description = list(), list(), list() + onset, duration, description, ch_names = list(), list(), list(), list() if annot_mask is not None: - on, dur = _mask_to_onsets_offsets(annot_mask.any(0)) - on = on / info['sfreq'] - dur = dur / info['sfreq'] - dur -= on - onset.extend(on) - duration.extend(dur) - description.extend(['BAD_NAN'] * len(on)) + assert annot_mask.shape[0] == len(info['ch_names']) + for mask, ch_name in zip(annot_mask, info['ch_names']): + on, dur = _mask_to_onsets_offsets(mask) + on = on / info['sfreq'] + dur = dur / info['sfreq'] + dur -= on + onset.extend(on) + duration.extend(dur) + description.extend(['BAD_SATURATED'] * len(on)) + ch_names.extend([[ch_name]] * len(on)) # Read triggers from event file if op.isfile(files['hdr'][:-3] + 'evt'): @@ -371,7 +377,8 @@ def prepend(li, str): onset.append(trigger_frame / samplingrate) duration.append(1.) # No duration info stored in files description.append(float(int(binary_value, 2))) - annot = Annotations(onset, duration, description) + ch_names.append(list()) + annot = Annotations(onset, duration, description, ch_names=ch_names) self.set_annotations(annot) def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 7bbcee01b65..0fa8c1c775c 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -104,7 +104,7 @@ def test_nirsport_v1_w_bad_sat(preload): raw = read_raw_nirx(fname, preload=preload) data = raw.get_data() assert not np.isnan(data).any() - assert len(raw.annotations) == 5 + assert len(raw.annotations) == 8 # annotated version and ignore should have same data but different annot raw_ignore = read_raw_nirx(fname, saturated='ignore', preload=preload) assert_allclose(raw_ignore.get_data(), data) @@ -117,7 +117,7 @@ def test_nirsport_v1_w_bad_sat(preload): assert not np.allclose(raw_nan.get_data(), data) raw_nan_annot = raw_ignore.copy() raw_nan_annot.set_annotations(annotate_nan(raw_nan)) - use_mask = np.where(raw.annotations.description == 'BAD_NAN') + use_mask = np.where(raw.annotations.description == 'BAD_SATURATED') for key in ('onset', 'duration'): a = getattr(raw_nan_annot.annotations, key) b = getattr(raw.annotations, key)[use_mask] diff --git a/mne/preprocessing/annotate_nan.py b/mne/preprocessing/annotate_nan.py index 8028ccba7b0..d7d7ef2991b 100644 --- a/mne/preprocessing/annotate_nan.py +++ b/mne/preprocessing/annotate_nan.py @@ -4,6 +4,7 @@ import numpy as np +from ..annotations import Annotations from ..utils import verbose from .artifact_detection import _annotations_from_mask @@ -21,9 +22,14 @@ def annotate_nan(raw, *, verbose=None): Returns ------- annot : instance of Annotations - Updated annotations for raw data. + New channel-specific annotations for the data. """ data, times = raw.get_data(return_times=True) - nans = np.any(np.isnan(data), axis=0) - annot = _annotations_from_mask(times, nans, 'BAD_NAN') + onsets, durations, ch_names = list(), list(), list() + for row, ch_name in zip(data, raw.ch_names): + annot = _annotations_from_mask(times, np.isnan(row), 'BAD_NAN') + onsets.extend(annot.onset) + durations.extend(annot.duration) + ch_names.extend([[ch_name]] * len(annot)) + annot = Annotations(onsets, durations, 'BAD_NAN', ch_names=ch_names) return annot From 01418c163dae5ce2dee48be60a33066c7867240d Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 6 May 2021 09:08:36 -0400 Subject: [PATCH 28/30] FIX: Paired --- mne/io/nirx/nirx.py | 9 ++++----- mne/io/nirx/tests/test_nirx.py | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 1ada7c55ce3..ec3d7f5de5b 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -345,8 +345,8 @@ def prepend(li, str): assert saturated == 'annotate' if annot_mask is None: annot_mask = np.zeros( - (len(info['ch_names']), last_sample + 1), bool) - annot_mask[ki::2] = mask + (len(info['ch_names']) // 2, last_sample + 1), bool) + annot_mask |= mask nan_mask[key] = None # shouldn't need again super(RawNIRX, self).__init__( @@ -356,8 +356,7 @@ def prepend(li, str): # make onset/duration/description onset, duration, description, ch_names = list(), list(), list(), list() if annot_mask is not None: - assert annot_mask.shape[0] == len(info['ch_names']) - for mask, ch_name in zip(annot_mask, info['ch_names']): + for ci, mask in enumerate(annot_mask): on, dur = _mask_to_onsets_offsets(mask) on = on / info['sfreq'] dur = dur / info['sfreq'] @@ -365,7 +364,7 @@ def prepend(li, str): onset.extend(on) duration.extend(dur) description.extend(['BAD_SATURATED'] * len(on)) - ch_names.extend([[ch_name]] * len(on)) + ch_names.extend([self.ch_names[2 * ci:2 * ci + 2]] * len(on)) # Read triggers from event file if op.isfile(files['hdr'][:-3] + 'evt'): diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 0fa8c1c775c..e4bbb3bdc3d 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -104,7 +104,7 @@ def test_nirsport_v1_w_bad_sat(preload): raw = read_raw_nirx(fname, preload=preload) data = raw.get_data() assert not np.isnan(data).any() - assert len(raw.annotations) == 8 + assert len(raw.annotations) == 5 # annotated version and ignore should have same data but different annot raw_ignore = read_raw_nirx(fname, saturated='ignore', preload=preload) assert_allclose(raw_ignore.get_data(), data) @@ -113,14 +113,14 @@ def test_nirsport_v1_w_bad_sat(preload): # nan version should not have same data, but we can give it the same annot raw_nan = read_raw_nirx(fname, saturated='nan', preload=preload) data_nan = raw_nan.get_data() - assert np.isnan(data_nan).any() # XXX should be a better accounting + assert np.isnan(data_nan).any() assert not np.allclose(raw_nan.get_data(), data) raw_nan_annot = raw_ignore.copy() raw_nan_annot.set_annotations(annotate_nan(raw_nan)) use_mask = np.where(raw.annotations.description == 'BAD_SATURATED') for key in ('onset', 'duration'): - a = getattr(raw_nan_annot.annotations, key) - b = getattr(raw.annotations, key)[use_mask] + a = getattr(raw_nan_annot.annotations, key)[::2] # one ch in each + b = getattr(raw.annotations, key)[use_mask] # two chs in each assert_allclose(a, b) From 94102c46b18cf8b8c537e00bc314ca4335b7943d Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 6 May 2021 09:09:21 -0400 Subject: [PATCH 29/30] FIX: No need to ignore --- mne/io/nirx/tests/test_nirx.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index e4bbb3bdc3d..ed92afc80d3 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -70,7 +70,6 @@ def test_nirsport_v1_wo_sat(): assert np.sum(np.isnan(data)) == 0 -@pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @requires_testing_data def test_nirsport_v1_w_sat(): @@ -94,7 +93,6 @@ def test_nirsport_v1_w_sat(): assert np.sum(np.isnan(data)) == 0 -@pytest.mark.filterwarnings('ignore:.*contains saturated data.*:') @pytest.mark.filterwarnings('ignore:.*Extraction of measurement.*:') @requires_testing_data @pytest.mark.parametrize('preload', (True, False)) From ac730cc1153d330442d233c370293fc27750af4a Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 6 May 2021 09:14:50 -0400 Subject: [PATCH 30/30] DOC: latest --- doc/changes/latest.inc | 6 +++++- doc/changes/names.inc | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index bc7fdc9e8ae..6b9d6f4cb79 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -19,11 +19,15 @@ Current (0.24.dev0) .. |New Contributor| replace:: **New Contributor** +.. |David Julien| replace:: **David Julien** + +.. |Romain Derollepot| replace:: **Romain Derollepot** + Enhancements ~~~~~~~~~~~~ .. - Add something cool (:gh:`9192` **by new contributor** |New Contributor|_) -- Nothing yet +- Add support for NIRSport devices to `mne.io.read_raw_nirx` (:gh:`9348` **by new contributor** |David Julien|_, **new contributor** |Romain Derollepot|_, `Robert Luke`_, and `Eric Larson`_) Bugs diff --git a/doc/changes/names.inc b/doc/changes/names.inc index 3c2e705a324..b03532a5dba 100644 --- a/doc/changes/names.inc +++ b/doc/changes/names.inc @@ -389,3 +389,7 @@ .. _Jack Zhang: https://github.com/jackz314 .. _Felix Klotzsche: https://github.com/eioe + +.. _David Julien: https://github.com/Swy7ch + +.. _Romain Derollepot: https://github.com/rderollepot