From 38921f93bbc6eeacccb42c598b0d126b538baa78 Mon Sep 17 00:00:00 2001 From: Marijn van Vliet Date: Tue, 7 Jan 2025 15:06:59 +0200 Subject: [PATCH 01/16] sph_harm -> sph_harm_y --- mne/preprocessing/maxwell.py | 6 +++--- mne/preprocessing/tests/test_maxwell.py | 6 +++--- mne/transforms.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mne/preprocessing/maxwell.py b/mne/preprocessing/maxwell.py index e1fb548caa5..47e5048f9e5 100644 --- a/mne/preprocessing/maxwell.py +++ b/mne/preprocessing/maxwell.py @@ -10,7 +10,7 @@ import numpy as np from scipy import linalg -from scipy.special import lpmv, sph_harm +from scipy.special import lpmv, sph_harm_y from .. import __version__ from .._fiff.compensator import make_compensator @@ -436,7 +436,7 @@ def _prep_maxwell_filter( # we purposefully stay away from shorthand notation in both and use # explicit terms (like 'azimuth' and 'polar') to avoid confusion. # See mathworld.wolfram.com/SphericalHarmonic.html for more discussion. - # Our code follows the same standard that ``scipy`` uses for ``sph_harm``. + # Our code follows the same standard that ``scipy`` uses for ``sph_harm_y``. # triage inputs ASAP to avoid late-thrown errors _validate_type(raw, BaseRaw, "raw") @@ -1487,7 +1487,7 @@ def _sss_basis_basic(exp, coils, mag_scale=100.0, method="standard"): S_in_out = list() grads_in_out = list() # Same spherical harmonic is used for both internal and external - sph = sph_harm(order, degree, az, pol) + sph = sph_harm_y(order, degree, az, pol) sph_norm = _sph_harm_norm(order, degree) # Compute complex gradient for all integration points # in spherical coordinates (Eq. 6). The gradient for rad, az, pol diff --git a/mne/preprocessing/tests/test_maxwell.py b/mne/preprocessing/tests/test_maxwell.py index 102900bb1fc..1a2e5012c8f 100644 --- a/mne/preprocessing/tests/test_maxwell.py +++ b/mne/preprocessing/tests/test_maxwell.py @@ -11,7 +11,7 @@ import pytest from numpy.testing import assert_allclose, assert_array_equal from scipy import sparse -from scipy.special import sph_harm +from scipy.special import sph_harm_y import mne from mne import compute_raw_covariance, concatenate_raws, pick_info, pick_types @@ -431,9 +431,9 @@ def test_spherical_conversions(): az, pol = np.meshgrid(np.linspace(0, 2 * np.pi, 30), np.linspace(0, np.pi, 20)) for degree in range(1, int_order): for order in range(0, degree + 1): - sph = sph_harm(order, degree, az, pol) + sph = sph_harm_y(order, degree, az, pol) # ensure that we satisfy the conjugation property - assert_allclose(_sh_negate(sph, order), sph_harm(-order, degree, az, pol)) + assert_allclose(_sh_negate(sph, order), sph_harm_y(-order, degree, az, pol)) # ensure our conversion functions work sph_real_pos = _sh_complex_to_real(sph, order) sph_real_neg = _sh_complex_to_real(sph, -order) diff --git a/mne/transforms.py b/mne/transforms.py index c85c31964b6..9914469b1a8 100644 --- a/mne/transforms.py +++ b/mne/transforms.py @@ -12,7 +12,7 @@ import numpy as np from scipy import linalg from scipy.spatial.distance import cdist -from scipy.special import sph_harm +from scipy.special import sph_harm_y from ._fiff.constants import FIFF from ._fiff.open import fiff_open @@ -926,7 +926,7 @@ def _compute_sph_harm(order, az, pol): # _deg_ord_idx(0, 0) = -1 so we're actually okay to use it here for degree in range(order + 1): for order_ in range(degree + 1): - sph = sph_harm(order_, degree, az, pol) + sph = sph_harm_y(order_, degree, az, pol) out[:, _deg_ord_idx(degree, order_)] = _sh_complex_to_real(sph, order_) if order_ > 0: out[:, _deg_ord_idx(degree, -order_)] = _sh_complex_to_real( From 35437db66bad76fef704e7fe4b47f6d2f7052ffe Mon Sep 17 00:00:00 2001 From: Marijn van Vliet Date: Tue, 7 Jan 2025 15:27:22 +0200 Subject: [PATCH 02/16] change the order of the parameter to match the new function --- mne/preprocessing/maxwell.py | 2 +- mne/preprocessing/tests/test_maxwell.py | 4 ++-- mne/transforms.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mne/preprocessing/maxwell.py b/mne/preprocessing/maxwell.py index 47e5048f9e5..2c7e4e84b00 100644 --- a/mne/preprocessing/maxwell.py +++ b/mne/preprocessing/maxwell.py @@ -1487,7 +1487,7 @@ def _sss_basis_basic(exp, coils, mag_scale=100.0, method="standard"): S_in_out = list() grads_in_out = list() # Same spherical harmonic is used for both internal and external - sph = sph_harm_y(order, degree, az, pol) + sph = sph_harm_y(degree, order, pol, az) sph_norm = _sph_harm_norm(order, degree) # Compute complex gradient for all integration points # in spherical coordinates (Eq. 6). The gradient for rad, az, pol diff --git a/mne/preprocessing/tests/test_maxwell.py b/mne/preprocessing/tests/test_maxwell.py index 1a2e5012c8f..0c9ec56b8c5 100644 --- a/mne/preprocessing/tests/test_maxwell.py +++ b/mne/preprocessing/tests/test_maxwell.py @@ -431,9 +431,9 @@ def test_spherical_conversions(): az, pol = np.meshgrid(np.linspace(0, 2 * np.pi, 30), np.linspace(0, np.pi, 20)) for degree in range(1, int_order): for order in range(0, degree + 1): - sph = sph_harm_y(order, degree, az, pol) + sph = sph_harm_y(degree, order, pol, az) # ensure that we satisfy the conjugation property - assert_allclose(_sh_negate(sph, order), sph_harm_y(-order, degree, az, pol)) + assert_allclose(_sh_negate(sph, order), sph_harm_y(degree, -order, pol, az)) # ensure our conversion functions work sph_real_pos = _sh_complex_to_real(sph, order) sph_real_neg = _sh_complex_to_real(sph, -order) diff --git a/mne/transforms.py b/mne/transforms.py index 9914469b1a8..b6632bbb41b 100644 --- a/mne/transforms.py +++ b/mne/transforms.py @@ -926,7 +926,7 @@ def _compute_sph_harm(order, az, pol): # _deg_ord_idx(0, 0) = -1 so we're actually okay to use it here for degree in range(order + 1): for order_ in range(degree + 1): - sph = sph_harm_y(order_, degree, az, pol) + sph = sph_harm_y(degree, order_, pol, az) out[:, _deg_ord_idx(degree, order_)] = _sh_complex_to_real(sph, order_) if order_ > 0: out[:, _deg_ord_idx(degree, -order_)] = _sh_complex_to_real( From 035726e02e721b38959acdd96a1bc07322253683 Mon Sep 17 00:00:00 2001 From: Marijn van Vliet Date: Tue, 7 Jan 2025 15:52:25 +0200 Subject: [PATCH 03/16] add docstring --- mne/fixes.py | 13 +++++++++++++ mne/preprocessing/maxwell.py | 4 ++-- mne/preprocessing/tests/test_maxwell.py | 2 +- mne/transforms.py | 3 +-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/mne/fixes.py b/mne/fixes.py index 2aed20492ec..070d4125d18 100644 --- a/mne/fixes.py +++ b/mne/fixes.py @@ -720,3 +720,16 @@ def minimum_phase(h, method="homomorphic", n_fft=None, *, half=True): n_out = (n_half + len(h) % 2) if half else len(h) return h_minimum[:n_out] + + +# SciPy 1.15 deprecates sph_harm for sph_harm_y and using it will trigger a +# DeprecationWarning. This is a backport of the new function for older SciPy versions. +def sph_harm_y(n, m, theta, phi, *, diff_n=0): + """Wrap scipy.special.sph_harm for sph_harm_y.""" + # Can be removed once we no longer support scipy < 1.15.0 + from scipy import special + + if "sph_harm_y" in special.__dict__: + return special.sph_harm_y(n, m, theta, phi, diff_n=diff_n) + else: + return special.sph_harm(m, n, phi, theta) diff --git a/mne/preprocessing/maxwell.py b/mne/preprocessing/maxwell.py index 2c7e4e84b00..789c8520f05 100644 --- a/mne/preprocessing/maxwell.py +++ b/mne/preprocessing/maxwell.py @@ -10,7 +10,7 @@ import numpy as np from scipy import linalg -from scipy.special import lpmv, sph_harm_y +from scipy.special import lpmv from .. import __version__ from .._fiff.compensator import make_compensator @@ -24,7 +24,7 @@ from ..annotations import _annotations_starts_stops from ..bem import _check_origin from ..channels.channels import _get_T1T2_mag_inds, fix_mag_coil_types -from ..fixes import _safe_svd, bincount +from ..fixes import _safe_svd, bincount, sph_harm_y from ..forward import _concatenate_coils, _create_meg_coils, _prep_meg_channels from ..io import BaseRaw, RawArray from ..surface import _normalize_vectors diff --git a/mne/preprocessing/tests/test_maxwell.py b/mne/preprocessing/tests/test_maxwell.py index 0c9ec56b8c5..f5e816258f8 100644 --- a/mne/preprocessing/tests/test_maxwell.py +++ b/mne/preprocessing/tests/test_maxwell.py @@ -11,7 +11,6 @@ import pytest from numpy.testing import assert_allclose, assert_array_equal from scipy import sparse -from scipy.special import sph_harm_y import mne from mne import compute_raw_covariance, concatenate_raws, pick_info, pick_types @@ -19,6 +18,7 @@ from mne.annotations import _annotations_starts_stops from mne.chpi import filter_chpi, read_head_pos from mne.datasets import testing +from mne.fixes import sph_harm_y from mne.forward import _prep_meg_channels, use_coil_def from mne.io import ( BaseRaw, diff --git a/mne/transforms.py b/mne/transforms.py index b6632bbb41b..7072ea25124 100644 --- a/mne/transforms.py +++ b/mne/transforms.py @@ -12,14 +12,13 @@ import numpy as np from scipy import linalg from scipy.spatial.distance import cdist -from scipy.special import sph_harm_y from ._fiff.constants import FIFF from ._fiff.open import fiff_open from ._fiff.tag import read_tag from ._fiff.write import start_and_end_file, write_coord_trans from .defaults import _handle_default -from .fixes import _get_img_fdata, jit +from .fixes import _get_img_fdata, jit, sph_harm_y from .utils import ( _check_fname, _check_option, From b231dba583f860322e9e5330ff19cc158e5d23d0 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 10:42:17 -0500 Subject: [PATCH 04/16] FIX: Green? --- mne/decoding/base.py | 6 +++++- mne/decoding/transformer.py | 8 ++++---- tools/install_pre_requirements.sh | 3 ++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mne/decoding/base.py b/mne/decoding/base.py index 85ed102b514..a291416bb17 100644 --- a/mne/decoding/base.py +++ b/mne/decoding/base.py @@ -85,7 +85,11 @@ def __sklearn_tags__(self): """Get sklearn tags.""" from sklearn.utils import get_tags # added in 1.6 - return get_tags(self.model) + # fit method below does not allow sparse data via check_data, we could + # eventually make it smarter if we had to + tags = get_tags(self.model) + tags.input_tags.sparse = False + return tags def __getattr__(self, attr): """Wrap to model for some attributes.""" diff --git a/mne/decoding/transformer.py b/mne/decoding/transformer.py index 8eb2dcc5510..e475cd22161 100644 --- a/mne/decoding/transformer.py +++ b/mne/decoding/transformer.py @@ -238,7 +238,7 @@ def inverse_transform(self, epochs_data): return out -class Vectorizer(TransformerMixin): +class Vectorizer(TransformerMixin, BaseEstimator): """Transform n-dimensional array into 2D array of n_samples by n_features. This class reshapes an n-dimensional array into an n_samples * n_features @@ -343,7 +343,7 @@ def inverse_transform(self, X): @fill_doc -class PSDEstimator(TransformerMixin): +class PSDEstimator(TransformerMixin, BaseEstimator): """Compute power spectral density (PSD) using a multi-taper method. Parameters @@ -452,7 +452,7 @@ def transform(self, epochs_data): @fill_doc -class FilterEstimator(TransformerMixin): +class FilterEstimator(TransformerMixin, BaseEstimator): """Estimator to filter RtEpochs. Applies a zero-phase low-pass, high-pass, band-pass, or band-stop @@ -743,7 +743,7 @@ def _apply_method(self, X, method): @fill_doc -class TemporalFilter(TransformerMixin): +class TemporalFilter(TransformerMixin, BaseEstimator): """Estimator to filter data array along the last dimension. Applies a zero-phase low-pass, high-pass, band-pass, or band-stop diff --git a/tools/install_pre_requirements.sh b/tools/install_pre_requirements.sh index 74868b0a435..9d670300a6f 100755 --- a/tools/install_pre_requirements.sh +++ b/tools/install_pre_requirements.sh @@ -18,10 +18,11 @@ python -m pip install $STD_ARGS pip setuptools packaging \ py-cpuinfo blosc2 hatchling echo "NumPy/SciPy/pandas etc." python -m pip uninstall -yq numpy +python -m pip install --upgrade matplotlib # TODO: Until https://github.com/matplotlib/matplotlib/pull/29427 lands python -m pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 \ --index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \ "numpy>=2.1.0.dev0" "scikit-learn>=1.6.dev0" "scipy>=1.15.0.dev0" \ - "pandas>=3.0.0.dev0" "matplotlib>=3.10.0.dev0" \ + "pandas>=3.0.0.dev0" \ "h5py>=3.12.1" "dipy>=1.10.0.dev0" "pyarrow>=19.0.0.dev0" "tables>=3.10.2.dev0" # statsmodels requires formulaic@main so we need to use --extra-index-url From 8f7ce76367c5e9dd23d26438ebdbea3f79333ca6 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 12:40:39 -0500 Subject: [PATCH 05/16] WIP: Check macOS --- .github/workflows/tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8975e72784b..9685dc88a00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -114,9 +114,6 @@ jobs: mamba nomkl if: ${{ !startswith(matrix.kind, 'pip') }} - # Make sure we have the right Python - - run: python -c "import platform; assert platform.machine() == 'arm64', platform.machine()" - if: matrix.os == 'macos-14' - run: ./tools/github_actions_dependencies.sh # Minimal commands on Linux (macOS stalls) - run: ./tools/get_minimal_commands.sh @@ -135,6 +132,8 @@ jobs: path: ~/mne_data - run: ./tools/github_actions_download.sh - run: ./tools/github_actions_test.sh + env: + MNE_IGNORE_WARNINGS_IN_TESTS: 1 - uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} From 39ce29d82079363594964a30992791fec8f98340 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 13:15:09 -0500 Subject: [PATCH 06/16] FIX: DS --- .github/workflows/tests.yml | 3 +++ doc/conf.py | 5 +++++ doc/development/contributing.rst | 2 +- doc/install/installers.rst | 2 +- mne/conftest.py | 4 ++++ mne/datasets/sleep_physionet/_utils.py | 7 ++++++- mne/export/tests/test_export.py | 2 +- 7 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9685dc88a00..ba23d92367d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -114,6 +114,9 @@ jobs: mamba nomkl if: ${{ !startswith(matrix.kind, 'pip') }} + # unknown error on macOS conda + - run: conda remove -y openmeeg + if: startswith(matrix.os, 'macos') && matrix.kind == 'mamba' - run: ./tools/github_actions_dependencies.sh # Minimal commands on Linux (macOS stalls) - run: ./tools/get_minimal_commands.sh diff --git a/doc/conf.py b/doc/conf.py index 7dd6ec90d4f..96028fb9045 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -666,6 +666,10 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): r"https://scholar.google.com/scholar\?cites=12188330066413208874&as_ylo=2014", r"https://scholar.google.com/scholar\?cites=1521584321377182930&as_ylo=2013", "https://www.research.chop.edu/imaging", + "http://prdownloads.sourceforge.net/optipng/optipng-0.7.8-win64.zip?download", + "https://sourceforge.net/projects/aespa/files/", + "https://sourceforge.net/projects/ezwinports/files/", + "https://www.mathworks.com/products/compiler/matlab-runtime.html", # 500 server error "https://openwetware.org/wiki/Beauchamp:FreeSurfer", # 503 Server error @@ -688,6 +692,7 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): # SSL problems sometimes "http://ilabs.washington.edu", "https://psychophysiology.cpmc.columbia.edu", + "https://erc.easme-web.eu", ] linkcheck_anchors = False # saves a bit of time linkcheck_timeout = 15 # some can be quite slow diff --git a/doc/development/contributing.rst b/doc/development/contributing.rst index 07c28f55d3d..011fd3c11f4 100644 --- a/doc/development/contributing.rst +++ b/doc/development/contributing.rst @@ -1114,6 +1114,6 @@ it can serve as a useful example of what to expect from the PR review process. .. optipng .. _optipng: http://optipng.sourceforge.net/ -.. _optipng for Windows: http://prdownloads.sourceforge.net/optipng/optipng-0.7.7-win32.zip?download +.. _optipng for Windows: http://prdownloads.sourceforge.net/optipng/optipng-0.7.8-win64.zip?download .. include:: ../links.inc diff --git a/doc/install/installers.rst b/doc/install/installers.rst index 5b7eeba5203..533c0207963 100644 --- a/doc/install/installers.rst +++ b/doc/install/installers.rst @@ -86,7 +86,7 @@ Platform-specific installers .. We have to use a button-link here because button-ref doesn't properly nested parse the inline code - .. button-link:: ./ides.html + .. button-link:: ides.html :ref-type: ref :color: success :shadow: diff --git a/mne/conftest.py b/mne/conftest.py index fc3bc3b7a53..6eea624467a 100644 --- a/mne/conftest.py +++ b/mne/conftest.py @@ -183,6 +183,10 @@ def pytest_configure(config): ignore:The (non_)?interactive_bk attribute was deprecated.*: # SWIG (via OpenMEEG) ignore:.*builtin type swigvarlink has no.*:DeprecationWarning + # eeglabio + ignore:numpy\.core\.records is deprecated.*:DeprecationWarning + # joblib + ignore:process .* is multi-threaded, use of fork/exec.*:DeprecationWarning """ # noqa: E501 for warning_line in warning_lines.split("\n"): warning_line = warning_line.strip() diff --git a/mne/datasets/sleep_physionet/_utils.py b/mne/datasets/sleep_physionet/_utils.py index b97d0611591..7fbcca3a2d7 100644 --- a/mne/datasets/sleep_physionet/_utils.py +++ b/mne/datasets/sleep_physionet/_utils.py @@ -2,6 +2,7 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. +import inspect import os import os.path as op @@ -114,12 +115,16 @@ def _update_sleep_temazepam_records(fname=TEMAZEPAM_SLEEP_RECORDS): data = data.set_index(("Subject - age - sex", "Nr")) data.index.name = "subject" data.columns.names = [None, None] + kwargs = dict() + # TODO VERSION can be removed once we require Pandas 2.1 + if "future_stack" in inspect.getfullargspec(pd.DataFrame.stack).args: + kwargs["future_stack"] = True data = ( data.set_index( [("Subject - age - sex", "Age"), ("Subject - age - sex", "M1/F2")], append=True, ) - .stack(level=0) + .stack(level=0, **kwargs) .reset_index() ) diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 706a83476e4..bb3787be8b9 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -476,7 +476,7 @@ def test_export_epochs_eeglab(tmp_path, preload): with ctx(): epochs.export(temp_fname) epochs.drop_channels([ch for ch in ["epoc", "STI 014"] if ch in epochs.ch_names]) - epochs_read = read_epochs_eeglab(temp_fname) + epochs_read = read_epochs_eeglab(temp_fname, verbose="error") # head radius assert epochs.ch_names == epochs_read.ch_names cart_coords = np.array([d["loc"][:3] for d in epochs.info["chs"]]) # just xyz cart_coords_read = np.array([d["loc"][:3] for d in epochs_read.info["chs"]]) From 59302141690e50b0c5a3aa88dfbc2c0955eb364f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 14:17:44 -0500 Subject: [PATCH 07/16] FIX: Rem --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba23d92367d..5c4a8c11187 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,7 +115,7 @@ jobs: nomkl if: ${{ !startswith(matrix.kind, 'pip') }} # unknown error on macOS conda - - run: conda remove -y openmeeg + - run: mamba remove -y openmeeg if: startswith(matrix.os, 'macos') && matrix.kind == 'mamba' - run: ./tools/github_actions_dependencies.sh # Minimal commands on Linux (macOS stalls) From 0e4d5f515d23a8ab8c5e37b5dc7da5597d8e49d9 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 15:00:07 -0500 Subject: [PATCH 08/16] WIP: Check PySide6 and SciPy --- environment.yml | 4 ++-- tools/hooks/update_environment_file.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 4dc45788af1..0b423c63f69 100644 --- a/environment.yml +++ b/environment.yml @@ -39,7 +39,7 @@ dependencies: - pyarrow - pybv - pymatreader - - PySide6 !=6.8.0,!=6.8.0.1 + - PySide6 !=6.8.1,!=6.8.0,!=6.8.0.1 - python-neo - python-picard - pyvista >=0.32,!=0.35.2,!=0.38.0,!=0.38.1,!=0.38.2,!=0.38.3,!=0.38.4,!=0.38.5,!=0.38.6,!=0.42.0 @@ -47,7 +47,7 @@ dependencies: - qdarkstyle !=3.2.2 - qtpy - scikit-learn - - scipy >=1.11 + - scipy >=1.11,<1.15.0 - sip - snirf - statsmodels diff --git a/tools/hooks/update_environment_file.py b/tools/hooks/update_environment_file.py index 8cac6193959..116c1e2277d 100755 --- a/tools/hooks/update_environment_file.py +++ b/tools/hooks/update_environment_file.py @@ -60,7 +60,9 @@ def split_dep(dep): # PySide6==6.7.0 only exists on PyPI, not conda-forge, so excluding it in # `environment.yaml` breaks the solver if package_name == "PySide6": - version_spec = version_spec.replace("!=6.7.0,", "") + version_spec = version_spec.replace("!=6.7.0,", "!=6.8.1,") + elif package_name == "scipy": + version_spec = f"{version_spec},<1.15.0" # rstrip output line in case `version_spec` == "" line = f" - {package_name} {version_spec}".rstrip() # use pip for packages needing e.g. `platform_system` or `python_version` triaging From 7fb3d4317c2e587a5da1774aaceb40973882c2ea Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 15:00:34 -0500 Subject: [PATCH 09/16] FIX: Restore --- .github/workflows/tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c4a8c11187..9685dc88a00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -114,9 +114,6 @@ jobs: mamba nomkl if: ${{ !startswith(matrix.kind, 'pip') }} - # unknown error on macOS conda - - run: mamba remove -y openmeeg - if: startswith(matrix.os, 'macos') && matrix.kind == 'mamba' - run: ./tools/github_actions_dependencies.sh # Minimal commands on Linux (macOS stalls) - run: ./tools/get_minimal_commands.sh From 9c2ab9195170dc7da6268311a7f848ed4b31eea6 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 15:07:55 -0500 Subject: [PATCH 10/16] FIX: Better? --- environment.yml | 2 +- tools/hooks/update_environment_file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 0b423c63f69..a362629686a 100644 --- a/environment.yml +++ b/environment.yml @@ -39,7 +39,7 @@ dependencies: - pyarrow - pybv - pymatreader - - PySide6 !=6.8.1,!=6.8.0,!=6.8.0.1 + - PySide6 <6.8.0 - python-neo - python-picard - pyvista >=0.32,!=0.35.2,!=0.38.0,!=0.38.1,!=0.38.2,!=0.38.3,!=0.38.4,!=0.38.5,!=0.38.6,!=0.42.0 diff --git a/tools/hooks/update_environment_file.py b/tools/hooks/update_environment_file.py index 116c1e2277d..4e645194b41 100755 --- a/tools/hooks/update_environment_file.py +++ b/tools/hooks/update_environment_file.py @@ -60,7 +60,7 @@ def split_dep(dep): # PySide6==6.7.0 only exists on PyPI, not conda-forge, so excluding it in # `environment.yaml` breaks the solver if package_name == "PySide6": - version_spec = version_spec.replace("!=6.7.0,", "!=6.8.1,") + version_spec = "<6.8.0" elif package_name == "scipy": version_spec = f"{version_spec},<1.15.0" # rstrip output line in case `version_spec` == "" From 56e6fd65350728d4fde54bab72a692fd79de81ce Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 16:08:30 -0500 Subject: [PATCH 11/16] FIX: Fixes --- .github/workflows/tests.yml | 13 +++---- environment.yml | 4 +-- mne/export/tests/test_export.py | 2 +- tools/github_actions_dependencies.sh | 2 +- tools/hooks/update_environment_file.py | 48 +++++--------------------- 5 files changed, 17 insertions(+), 52 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9685dc88a00..1d294c638e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,19 +99,18 @@ jobs: python-version: ${{ matrix.python }} if: startswith(matrix.kind, 'pip') # Python (if conda) - - name: Remove numba and dipy + - name: Fixes for conda run: | # TODO: Remove when numba 0.59 and dipy 1.8 land on conda-forge - sed -i '/numba/d' environment.yml - sed -i '/dipy/d' environment.yml - sed -i 's/- mne$/- mne-base/' environment.yml - if: matrix.os == 'ubuntu-latest' && startswith(matrix.kind, 'conda') && matrix.python == '3.12' + if [[ "$RUNNER_OS" == "macOS" ]]; then + sed -i "s/ - PySide6 .*/ - PySide6 <6.8/g" environment.yml + fi + if: matrix.kind == 'conda' || matrix.kind == 'mamba' - uses: mamba-org/setup-micromamba@v2 with: environment-file: ${{ env.CONDA_ENV }} environment-name: mne create-args: >- python=${{ env.PYTHON_VERSION }} - mamba nomkl if: ${{ !startswith(matrix.kind, 'pip') }} - run: ./tools/github_actions_dependencies.sh @@ -132,8 +131,6 @@ jobs: path: ~/mne_data - run: ./tools/github_actions_download.sh - run: ./tools/github_actions_test.sh - env: - MNE_IGNORE_WARNINGS_IN_TESTS: 1 - uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/environment.yml b/environment.yml index a362629686a..4dc45788af1 100644 --- a/environment.yml +++ b/environment.yml @@ -39,7 +39,7 @@ dependencies: - pyarrow - pybv - pymatreader - - PySide6 <6.8.0 + - PySide6 !=6.8.0,!=6.8.0.1 - python-neo - python-picard - pyvista >=0.32,!=0.35.2,!=0.38.0,!=0.38.1,!=0.38.2,!=0.38.3,!=0.38.4,!=0.38.5,!=0.38.6,!=0.42.0 @@ -47,7 +47,7 @@ dependencies: - qdarkstyle !=3.2.2 - qtpy - scikit-learn - - scipy >=1.11,<1.15.0 + - scipy >=1.11 - sip - snirf - statsmodels diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index bb3787be8b9..ca0853837fc 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -145,7 +145,7 @@ def _create_raw_for_edf_tests(stim_channel_index=None): edfio_mark = pytest.mark.skipif( - not _check_edfio_installed(strict=False), reason="unsafe use of private module" + not _check_edfio_installed(strict=False), reason="requires edfio" ) diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index 149f5a194da..cebd2caefa7 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -23,7 +23,7 @@ if [ ! -z "$CONDA_ENV" ]; then elif [[ "${MNE_CI_KIND}" == "pip" ]]; then # Only used for 3.13 at the moment, just get test deps plus a few extras # that we know are available - INSTALL_ARGS="nibabel scikit-learn numpydoc PySide6 mne-qt-browser" + INSTALL_ARGS="nibabel scikit-learn numpydoc PySide6 mne-qt-browser pandas h5io mffpy defusedxml" INSTALL_KIND="test" else test "${MNE_CI_KIND}" == "pip-pre" diff --git a/tools/hooks/update_environment_file.py b/tools/hooks/update_environment_file.py index 4e645194b41..78df3eb5f9c 100755 --- a/tools/hooks/update_environment_file.py +++ b/tools/hooks/update_environment_file.py @@ -4,14 +4,14 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -import difflib import re from pathlib import Path import tomllib repo_root = Path(__file__).resolve().parents[2] -pyproj = tomllib.loads((repo_root / "pyproject.toml").read_text("utf-8")) +with open(repo_root / "pyproject.toml", "rb") as fid: + pyproj = tomllib.load(fid) # Get our "full" dependences from `pyproject.toml`, but actually ignore the # "full" section as it's just "full-noqt" plus PyQt6, and for conda we need PySide @@ -48,11 +48,6 @@ def split_dep(dep): translations = dict(neo="python-neo") pip_deps = set() conda_deps = set() -check_old = ( - "numpy scipy matplotlib pandas scikit-learn nibabel tqdm pooch decorator " - "packaging jinja2 lazy_loader" -).split() -old_deps = [None] * len(check_old) for dep in deps: package_name, version_spec = split_dep(dep) # handle package name differences @@ -60,9 +55,7 @@ def split_dep(dep): # PySide6==6.7.0 only exists on PyPI, not conda-forge, so excluding it in # `environment.yaml` breaks the solver if package_name == "PySide6": - version_spec = "<6.8.0" - elif package_name == "scipy": - version_spec = f"{version_spec},<1.15.0" + version_spec = version_spec.replace("!=6.7.0,", "") # rstrip output line in case `version_spec` == "" line = f" - {package_name} {version_spec}".rstrip() # use pip for packages needing e.g. `platform_system` or `python_version` triaging @@ -70,12 +63,6 @@ def split_dep(dep): pip_deps.add(f" {line}") else: conda_deps.add(line) - # old deps - if package_name in check_old: - # Pull out >= part, change to =, remove < (which should be after comma) - old_deps[check_old.index(package_name)] = line.replace(">=", "=").split(",")[0] -for di, dep in enumerate(old_deps): - assert dep is not None, f"Missing {check_old[di]}" # TODO: temporary workaround while we wait for a release containing the fix for # https://github.com/mamba-org/mamba/issues/3467 @@ -89,33 +76,14 @@ def split_dep(dep): """ pip_section = pip_section if len(pip_deps) else "" # prepare the env file -header = f"""\ +env = f"""\ # THIS FILE IS AUTO-GENERATED BY {'/'.join(Path(__file__).parts[-3:])} AND WILL BE OVERWRITTEN name: mne channels: - conda-forge -dependencies:""" # noqa: E501 -env = f"""{header} +dependencies: - python {req_python} {newline.join(sorted(conda_deps, key=str.casefold))} -{pip_section}""" - -env_file = repo_root / "environment.yml" -old_env = env_file.read_text("utf-8") -if old_env != env: - diff = "\n".join(difflib.unified_diff(old_env.splitlines(), env.splitlines())) - print(f"Updating {env_file} with diff:\n{diff}") - env_file.write_text(env, encoding="utf-8") - -# Now we also updated tools/environment_old.yml -env_file = repo_root / "tools" / "environment_old.yml" -old_env = env_file.read_text("utf-8") -use_python = req_python.replace(">=", "=") -env = f"""{header} - - python {use_python} -{newline.join(old_deps)} -""" -if old_env != env: - diff = "\n".join(difflib.unified_diff(old_env.splitlines(), env.splitlines())) - print(f"Updating {env_file} with diff:\n{diff}") - env_file.write_text(env, encoding="utf-8") +{pip_section}""" # noqa: E501 + +(repo_root / "environment.yml").write_text(env) From 1af857d07441fe36d7f0c51804f6d0171aed79d1 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 16:11:49 -0500 Subject: [PATCH 12/16] FIX: Better --- .github/workflows/tests.yml | 1 - environment.yml | 2 ++ tools/hooks/update_environment_file.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d294c638e8..69632fe68c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,7 +111,6 @@ jobs: environment-name: mne create-args: >- python=${{ env.PYTHON_VERSION }} - nomkl if: ${{ !startswith(matrix.kind, 'pip') }} - run: ./tools/github_actions_dependencies.sh # Minimal commands on Linux (macOS stalls) diff --git a/environment.yml b/environment.yml index 4dc45788af1..b25d83958c2 100644 --- a/environment.yml +++ b/environment.yml @@ -23,11 +23,13 @@ dependencies: - joblib - jupyter - lazy_loader >=0.3 + - mamba - matplotlib >=3.7 - mffpy >=0.5.7 - mne-qt-browser - nibabel - nilearn + - nomkl - numba - numpy >=1.25,<3 - openmeeg >=2.5.5 diff --git a/tools/hooks/update_environment_file.py b/tools/hooks/update_environment_file.py index 78df3eb5f9c..bc36567147a 100755 --- a/tools/hooks/update_environment_file.py +++ b/tools/hooks/update_environment_file.py @@ -22,7 +22,7 @@ deps |= set(section_deps) recursive_deps = set(d for d in deps if d.startswith("mne[")) deps -= recursive_deps -deps |= {"pip"} +deps |= {"pip", "mamba", "nomkl"} def remove_spaces(version_spec): From 1a7844a0565edc82ade12766261a336e1e61c291 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 7 Jan 2025 16:12:34 -0500 Subject: [PATCH 13/16] TST: More [circle linkcheck] From 9ef1f99e6ed01ca531928667841887bfbf01a3bc Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 8 Jan 2025 09:57:17 -0500 Subject: [PATCH 14/16] FIX: SED --- .github/workflows/tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 69632fe68c4..0fc130bba97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -100,9 +100,12 @@ jobs: if: startswith(matrix.kind, 'pip') # Python (if conda) - name: Fixes for conda - run: | # TODO: Remove when numba 0.59 and dipy 1.8 land on conda-forge - if [[ "$RUNNER_OS" == "macOS" ]]; then - sed -i "s/ - PySide6 .*/ - PySide6 <6.8/g" environment.yml + run: | + # For some reason on Linux we get crashes + if [[ "$RUNNER_OS" == "Linux" ]]; then + sed -i "/numba/d" environment.yml + elif [[ "$RUNNER_OS" == "macOS" ]]; then + sed -i "" "s/ - PySide6 .*/ - PySide6 <6.8/g" environment.yml fi if: matrix.kind == 'conda' || matrix.kind == 'mamba' - uses: mamba-org/setup-micromamba@v2 From 67fbf32d8fdd0596d75f4fa44de274e944ea8600 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 8 Jan 2025 11:05:40 -0500 Subject: [PATCH 15/16] FIX: Missed --- tools/install_pre_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/install_pre_requirements.sh b/tools/install_pre_requirements.sh index 9d670300a6f..113f122b399 100755 --- a/tools/install_pre_requirements.sh +++ b/tools/install_pre_requirements.sh @@ -49,7 +49,7 @@ python -m pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https:/ python -c "import vtk" echo "PyVista" -python -m pip install $STD_ARGS "git+https://github.com/pyvista/pyvista" +python -m pip install $STD_ARGS "git+https://github.com/pyvista/pyvista" trame trame-vtk trame-vuetify echo "picard" python -m pip install $STD_ARGS git+https://github.com/pierreablin/picard @@ -58,7 +58,7 @@ echo "pyvistaqt" pip install $STD_ARGS git+https://github.com/pyvista/pyvistaqt echo "imageio-ffmpeg, xlrd, mffpy" -pip install $STD_ARGS imageio-ffmpeg xlrd mffpy traitlets pybv eeglabio +pip install $STD_ARGS imageio-ffmpeg xlrd mffpy traitlets pybv eeglabio defusedxml echo "mne-qt-browser" pip install $STD_ARGS git+https://github.com/mne-tools/mne-qt-browser From 6e4a156146a372dac872fc99f0b8f1c7800ef430 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 8 Jan 2025 11:09:47 -0500 Subject: [PATCH 16/16] FIX: Grep --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 644fd8b31b7..4297dc5fedf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -252,7 +252,7 @@ jobs: name: Check sphinx log for warnings (which are treated as errors) when: always command: | - ! grep "^.* WARNING: .*$" sphinx_log.txt + ! grep "^.* (WARNING|ERROR): .*$" sphinx_log.txt - run: name: Show profiling output when: always