Fix read_raw_eyelink() failure when recording blocks are empty or starting with empty values

doc/changes/dev/13571.bugfix.rst

@@ -0,0 +1 @@
+Fix bug where :func:`mne.io.read_raw_eyelink` fails to handle data starting with empty blocks or blocks starting with missing values, by :newcontrib:`Varun Kasyap Pentamaraju` and `Qian Chu`_.

mne/io/eyelink/_utils.py

@@ -103,7 +103,7 @@ def _parse_recording_blocks(fname):
     samples lines start with a posix-like string,
     and contain eyetracking sample info. Event Lines
     start with an upper case string and contain info
-    about occular events (i.e. blink/saccade), or experiment
+    about ocular events (i.e. blink/saccade), or experiment
     messages sent by the stimulus presentation software.
     """
     with fname.open() as file:
@@ -182,14 +182,14 @@ def _validate_data(data_blocks: list):
         pupil_units.append(block["info"]["pupil_unit"])
     if "GAZE" in units:
         logger.info(
-            "Pixel coordinate data detected."
+            "Pixel coordinate data detected. "
             "Pass `scalings=dict(eyegaze=1e3)` when using plot"
             " method to make traces more legible."
         )
     if "HREF" in units:
         logger.info("Head-referenced eye-angle (HREF) data detected.")
     elif "PUPIL" in units:
-        warn("Raw eyegaze coordinates detected. Analyze with caution.")
+        warn("Raw pupil position data detected. Analyze with caution.")
     if "AREA" in pupil_units:
         logger.info("Pupil-size area detected.")
     elif "DIAMETER" in pupil_units:
@@ -369,7 +369,7 @@ def _create_dataframes_for_block(block, apply_offsets):
         df_dict["samples"] = pd.DataFrame(block["samples"])
         df_dict["samples"] = _drop_status_col(df_dict["samples"])  # drop STATUS col
 
-    # dataframe for each type of occular event in this block
+    # dataframe for each type of ocular event in this block
     for event, label in zip(
         ["EFIX", "ESACC", "EBLINK"], ["fixations", "saccades", "blinks"]
     ):
@@ -559,11 +559,22 @@ def _drop_status_col(samples_df):
     status_cols = []
     # we know the first 3 columns will be the time, xpos, ypos
     for col in samples_df.columns[3:]:
-        if samples_df[col][0][0].isnumeric():
-            # if the value is numeric, it's not a status column
-            continue
-        if len(samples_df[col][0]) in [3, 5, 13, 17]:
+        # use first valid index and value to ignore leading empty values
+        # see https://github.com/mne-tools/mne-python/issues/13567
+        first_valid_index = samples_df[col].first_valid_index()
+        if first_valid_index is None:
+            # The entire column is NaN, so we can drop it
             status_cols.append(col)
+            continue
+        first_value = samples_df.loc[first_valid_index, col]
+        try:
+            float(first_value)
+            continue  # if the value is numeric, it's not a status column
+        except (ValueError, TypeError):
+            # cannot convert to float, so it might be a status column
+            # further check the length of the string value
+            if len(first_value) in [3, 5, 13, 17]:
+                status_cols.append(col)
     return samples_df.drop(columns=status_cols)
 
 
@@ -697,7 +708,7 @@ def _adjust_times(
     -----
     After _parse_recording_blocks, Files with multiple recording blocks will
     have missing timestamps for the duration of the period between the blocks.
-    This would cause the occular annotations (i.e. blinks) to not line up with
+    This would cause the ocular annotations (i.e. blinks) to not line up with
     the signal.
     """
     pd = _check_pandas_installed()
@@ -723,7 +734,7 @@ def _find_overlaps(df, max_time=0.05):
     Parameters
     ----------
     df : pandas.DataFrame
-        Pandas DataFrame with occular events (fixations, saccades, blinks)
+        Pandas DataFrame with ocular events (fixations, saccades, blinks)
     max_time : float (default 0.05)
         Time in seconds. Defaults to .05 (50 ms)
 

mne/io/eyelink/eyelink.py

@@ -69,7 +69,7 @@ def read_raw_eyelink(
 
 @fill_doc
 class RawEyelink(BaseRaw):
-    """Raw object from an XXX file.
+    """Raw object from an Eyelink file.
 
     Parameters
     ----------

mne/io/eyelink/tests/test_eyelink.py

@@ -172,7 +172,7 @@ def test_fill_times(fname):
 
 
 def test_find_overlaps():
-    """Test finding overlapping occular events between the left and right eyes.
+    """Test finding overlapping ocular events between the left and right eyes.
 
     In the simulated blink df below, the first two rows
     will be considered an overlap because the diff() of both the 'time' and
@@ -360,7 +360,13 @@ def _simulate_eye_tracking_data(in_file, out_file):
         fp.write("START\t7452389\tRIGHT\tSAMPLES\tEVENTS\n")
         fp.write(f"{new_samples_line}\n")
 
-        for timestamp in np.arange(7452389, 7453390):  # simulate a second block
+        # simulate a second block that starts with empty samples
+        for timestamp in np.arange(7452389, 7453154):
+            # samples without last status column
+            # see https://github.com/mne-tools/mne-python/issues/13567
+            fp.write(f"{timestamp}\t.\t.\t0.0\t.\t.\t.\t.\t0.0\t...\t.\t.\t.\n")
+
+        for timestamp in np.arange(7453154, 7453390):  # second block continues
             fp.write(
                 f"{timestamp}\t-2434.0\t-1760.0\t840.0\t100\t20\t45\t45\t127.0\t"
                 "...\t1497\t5189\t512.5\t.............\n"
@@ -382,13 +388,22 @@ def _simulate_eye_tracking_data(in_file, out_file):
                     "1497.0\t5189.0\t512.5\t.............\n"
                 )
                 button_idx += 1
-        fp.write("END\t7453390\tRIGHT\tSAMPLES\tEVENTS\n")
+        fp.write("END\t7453490\tRIGHT\tSAMPLES\tEVENTS\n")
+
+        # simulate a third block with no samples at all
+        fp.write("START\t7454490\tRIGHT\tSAMPLES\tEVENTS\n")
+        fp.write(f"{new_samples_line}\n")
+        for timestamp in np.arange(7454490, 7454990):
+            fp.write(
+                f"{timestamp}\t.\t.\t0.0\t.\t.\t.\t.\t0.0\t...\t.\t.\t.\n"  # All empty
+            )
+        fp.write("END\t7454990\tRIGHT\tSAMPLES\tEVENTS\n")
 
 
 @requires_testing_data
 @pytest.mark.parametrize("fname", [fname_href])
 def test_multi_block_misc_channels(fname, tmp_path):
-    """Test a file with many edge casses.
+    """Test a file with many edge cases.
 
     This file has multiple acquisition blocks, each tracking a different eye.
     The coordinates are in raw units (not pixels or radians).
@@ -399,7 +414,7 @@ def test_multi_block_misc_channels(fname, tmp_path):
 
     with (
         _record_warnings(),
-        pytest.warns(RuntimeWarning, match="Raw eyegaze coordinates"),
+        pytest.warns(RuntimeWarning, match="Raw pupil position data detected"),
         pytest.warns(RuntimeWarning, match="The eye being tracked changed"),
     ):
         raw = read_raw_eyelink(out_file, apply_offsets=True)
@@ -426,15 +441,15 @@ def test_multi_block_misc_channels(fname, tmp_path):
     assert raw.ch_names == chs_in_file
     assert raw.annotations.description[1] == "SYNCTIME"
 
-    assert raw.annotations.description[-7] == "BAD_ACQ_SKIP"
-    assert np.isclose(raw.annotations.onset[-7], 1.001)
-    assert np.isclose(raw.annotations.duration[-7], 0.1)
+    assert raw.annotations.description[-8] == "BAD_ACQ_SKIP"
+    assert np.isclose(raw.annotations.onset[-8], 1.001)
+    assert np.isclose(raw.annotations.duration[-8], 0.1)
 
     data, times = raw.get_data(return_times=True)
     assert not np.isnan(data[0, np.where(times < 1)[0]]).any()
     assert np.isnan(data[0, np.logical_and(times > 1, times <= 1.1)]).all()
 
-    assert raw.annotations.description[-6] == "button_1_press"
+    assert raw.annotations.description[-7] == "button_1_press"
     button_idx = [
         ii
         for ii, desc in enumerate(raw.annotations.description)
@@ -443,6 +458,9 @@ def test_multi_block_misc_channels(fname, tmp_path):
     assert len(button_idx) == 6
     assert_allclose(raw.annotations.onset[button_idx[0]], 2.102, atol=1e-3)
 
+    assert raw.annotations.description[-1] == "BAD_ACQ_SKIP"
+    assert_allclose(raw.annotations.duration[-1], 1.0, atol=1e-3)
+
     # smoke test for reading events with missing samples (should not emit a warning)
     find_events(raw, verbose=True)
 

mne/utils/docs.py

@@ -1599,10 +1599,10 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75):
 
 docdict["eyelink_create_annotations"] = """
 create_annotations : bool | list (default True)
-    Whether to create :class:`~mne.Annotations` from occular events
+    Whether to create :class:`~mne.Annotations` from ocular events
     (blinks, fixations, saccades) and experiment messages. If a list, must
     contain one or more of ``['fixations', 'saccades',' blinks', messages']``.
-    If True, creates :class:`~mne.Annotations` for both occular events and
+    If True, creates :class:`~mne.Annotations` for both ocular events and
     experiment messages.
 """