From f67e65749dd19a478e819fb6e96c519a11ba8e01 Mon Sep 17 00:00:00 2001 From: jCalderTravis <38797399+jCalderTravis@users.noreply.github.com> Date: Sat, 18 Mar 2023 12:28:41 +0100 Subject: [PATCH 1/5] Additional test for one-tailed permutation t-test --- mne/stats/tests/test_permutations.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mne/stats/tests/test_permutations.py b/mne/stats/tests/test_permutations.py index 2643961c5f2..677bae2de35 100644 --- a/mne/stats/tests/test_permutations.py +++ b/mne/stats/tests/test_permutations.py @@ -58,10 +58,14 @@ def test_permutation_t_test(): assert_allclose(p_values_clust, p_values[keep], atol=1e-2) X = np.random.randn(18, 1) - t_obs, p_values, H0 = permutation_t_test(X, n_permutations='all') - t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0) - assert_allclose(t_obs[0], t_obs_scipy, 8) - assert_allclose(p_values[0], p_values_scipy, rtol=1e-2) + tail_codes = {'two-sided': 0, 'less': -1, 'greater': 1} + for this_tail, this_code in tail_codes.items(): + t_obs, p_values, H0 = permutation_t_test(X, n_permutations='all', + tail=this_code) + t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0, + alternative=this_tail) + assert_allclose(t_obs[0], t_obs_scipy, 8) + assert_allclose(p_values[0], p_values_scipy, rtol=1e-2) def test_ci(): From 34598148fa5401a47199ec0b97a1b801899c2caf Mon Sep 17 00:00:00 2001 From: jCalderTravis <38797399+jCalderTravis@users.noreply.github.com> Date: Tue, 21 Mar 2023 10:54:20 +0100 Subject: [PATCH 2/5] Clearer checking of all tails Co-authored-by: Eric Larson --- mne/stats/tests/test_permutations.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mne/stats/tests/test_permutations.py b/mne/stats/tests/test_permutations.py index 677bae2de35..d639a44e035 100644 --- a/mne/stats/tests/test_permutations.py +++ b/mne/stats/tests/test_permutations.py @@ -58,14 +58,15 @@ def test_permutation_t_test(): assert_allclose(p_values_clust, p_values[keep], atol=1e-2) X = np.random.randn(18, 1) - tail_codes = {'two-sided': 0, 'less': -1, 'greater': 1} - for this_tail, this_code in tail_codes.items(): - t_obs, p_values, H0 = permutation_t_test(X, n_permutations='all', - tail=this_code) - t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0, - alternative=this_tail) - assert_allclose(t_obs[0], t_obs_scipy, 8) - assert_allclose(p_values[0], p_values_scipy, rtol=1e-2) +@pytest.mark.parametrize('tail', [0, -1, 1]) +def test_permutation_t_test_tail(tail): + """Test that tails work properly.""" + t_obs, p_values, H0 = permutation_t_test(X, n_permutations='all', + tail=this_code) + t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0, + alternative=this_tail) + assert_allclose(t_obs[0], t_obs_scipy, 8) + assert_allclose(p_values[0], p_values_scipy, rtol=1e-2) def test_ci(): From c3d45265afee821b2d6dba01f7da97a575ff4345 Mon Sep 17 00:00:00 2001 From: jCalderTravis <38797399+jCalderTravis@users.noreply.github.com> Date: Tue, 21 Mar 2023 12:13:40 +0100 Subject: [PATCH 3/5] Adjusting parameterisation of test --- mne/stats/tests/test_permutations.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mne/stats/tests/test_permutations.py b/mne/stats/tests/test_permutations.py index d639a44e035..d6bf981166d 100644 --- a/mne/stats/tests/test_permutations.py +++ b/mne/stats/tests/test_permutations.py @@ -5,6 +5,7 @@ from numpy.testing import assert_array_equal, assert_allclose import numpy as np from scipy import stats, sparse +import pytest from mne.stats import permutation_cluster_1samp_test from mne.stats.permutations import (permutation_t_test, _ci, @@ -57,14 +58,17 @@ def test_permutation_t_test(): assert_allclose(t_obs_clust, t_obs) assert_allclose(p_values_clust, p_values[keep], atol=1e-2) - X = np.random.randn(18, 1) -@pytest.mark.parametrize('tail', [0, -1, 1]) -def test_permutation_t_test_tail(tail): + +@pytest.mark.parametrize('tail_name,tail_code', + [('two-sided', 0), ('less', -1), ('greater', 1)]) +def test_permutation_t_test_tail(tail_name, tail_code): """Test that tails work properly.""" - t_obs, p_values, H0 = permutation_t_test(X, n_permutations='all', - tail=this_code) + X = np.random.randn(18, 1) + + t_obs, p_values, _ = permutation_t_test(X, n_permutations='all', + tail=tail_code) t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0, - alternative=this_tail) + alternative=tail_name) assert_allclose(t_obs[0], t_obs_scipy, 8) assert_allclose(p_values[0], p_values_scipy, rtol=1e-2) From c5b0d4fe7d60f5c66367178515e712202d190b96 Mon Sep 17 00:00:00 2001 From: jCalderTravis <38797399+jCalderTravis@users.noreply.github.com> Date: Wed, 24 May 2023 11:00:15 +0200 Subject: [PATCH 4/5] Marking fail as expected --- mne/stats/tests/test_permutations.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mne/stats/tests/test_permutations.py b/mne/stats/tests/test_permutations.py index ecdcaba0e82..ab265dc12f0 100644 --- a/mne/stats/tests/test_permutations.py +++ b/mne/stats/tests/test_permutations.py @@ -58,7 +58,11 @@ def test_permutation_t_test(): @pytest.mark.parametrize('tail_name,tail_code', - [('two-sided', 0), ('less', -1), ('greater', 1)]) + [('two-sided', 0), + pytest.param('less', -1, marks=pytest.mark.xfail( + reason="Bug in permutation function")), + pytest.param('greater', 1, marks=pytest.mark.xfail( + reason="Bug in permutation function"))]) def test_permutation_t_test_tail(tail_name, tail_code): """Test that tails work properly.""" X = np.random.randn(18, 1) From 0029a6b223f263158e4f4cf4f3535aa82c216de5 Mon Sep 17 00:00:00 2001 From: jCalderTravis <38797399+jCalderTravis@users.noreply.github.com> Date: Wed, 24 May 2023 11:08:24 +0200 Subject: [PATCH 5/5] Formatting with Black --- mne/stats/tests/test_permutations.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/mne/stats/tests/test_permutations.py b/mne/stats/tests/test_permutations.py index ab265dc12f0..245ac140182 100644 --- a/mne/stats/tests/test_permutations.py +++ b/mne/stats/tests/test_permutations.py @@ -57,20 +57,24 @@ def test_permutation_t_test(): assert_allclose(p_values_clust, p_values[keep], atol=1e-2) -@pytest.mark.parametrize('tail_name,tail_code', - [('two-sided', 0), - pytest.param('less', -1, marks=pytest.mark.xfail( - reason="Bug in permutation function")), - pytest.param('greater', 1, marks=pytest.mark.xfail( - reason="Bug in permutation function"))]) +@pytest.mark.parametrize( + "tail_name,tail_code", + [ + ("two-sided", 0), + pytest.param( + "less", -1, marks=pytest.mark.xfail(reason="Bug in permutation function") + ), + pytest.param( + "greater", 1, marks=pytest.mark.xfail(reason="Bug in permutation function") + ), + ], +) def test_permutation_t_test_tail(tail_name, tail_code): """Test that tails work properly.""" X = np.random.randn(18, 1) - t_obs, p_values, _ = permutation_t_test(X, n_permutations='all', - tail=tail_code) - t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0, - alternative=tail_name) + t_obs, p_values, _ = permutation_t_test(X, n_permutations="all", tail=tail_code) + t_obs_scipy, p_values_scipy = stats.ttest_1samp(X[:, 0], 0, alternative=tail_name) assert_allclose(t_obs[0], t_obs_scipy, 8) assert_allclose(p_values[0], p_values_scipy, rtol=1e-2)