From 8bd166091d5d2b697c7c77cb76e53dc5c93b964e Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 12:18:54 -0500 Subject: [PATCH 1/8] MAINT: Skip matplotlib pre for now --- azure-pipelines.yml | 32 ++-------------------------- tools/azure_dependencies.sh | 26 ++++++++++++++++++++++ tools/github_actions_dependencies.sh | 3 ++- 3 files changed, 30 insertions(+), 31 deletions(-) create mode 100755 tools/azure_dependencies.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 95625aa4d0d..a2b271ca67f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -213,9 +213,6 @@ stages: # https://github.com/MicrosoftDocs/pipelines-anaconda # Qt 5.9 post-link failure: # https://github.com/ContinuumIO/anaconda-issues/issues/10949 - - script: echo "##vso[task.prependpath]%CONDA%;%CONDA%\condabin;%CONDA%\Scripts;%CONDA%\Library\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;" - condition: in(variables['TEST_MODE'], 'conda') - displayName: Add conda to PATH, deal with Qt 5.9 bug - bash: | set -e git clone --depth 1 git://github.com/pyvista/gl-ci-helpers.git @@ -223,33 +220,8 @@ stages: displayName: Install OpenGL - bash: | set -e - python -m pip install --upgrade pip setuptools - python -m pip install --upgrade numpy scipy vtk - python -m pip install --upgrade --use-deprecated=legacy-resolver --only-binary="numba,llvmlite" -r requirements.txt -r requirements_testing.txt - python -m pip install codecov - condition: eq(variables['TEST_MODE'], 'pip') - displayName: 'Install dependencies with pip' - - bash: | - set -e - python -m pip install --upgrade pip setuptools - python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy - python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" scipy pandas scikit-learn matplotlib h5py Pillow - python -m pip install --upgrade --only-binary vtk vtk; - python -m pip install https://github.com/pyvista/pyvista/zipball/master - python -m pip install https://github.com/pyvista/pyvistaqt/zipball/master - python -m pip install --use-deprecated=legacy-resolver --only-binary="numba,llvmlite" -r requirements.txt -r requirements_testing.txt codecov - condition: eq(variables['TEST_MODE'], 'pre-pip') - displayName: 'Install dependencies with pip --pre' - - powershell: | - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - $PSDefaultParameterValues['*:ErrorAction']='Stop' - conda update -n base -c defaults conda - conda env update --name base --file environment.yml - pip uninstall -yq mne - pip install -r requirements_testing.txt codecov - condition: eq(variables['TEST_MODE'], 'conda') - displayName: 'Install dependencies with conda' + ./tools/azure_dependencies.sh + displayName: 'Install dependencies' - script: python setup.py develop displayName: 'Install MNE-Python dev' - script: mne sys_info diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh new file mode 100755 index 00000000000..056ba4e9507 --- /dev/null +++ b/tools/azure_dependencies.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ef + +if [ "${TEST_MODE}" == "conda" ]; then + echo "##vso[task.prependpath]%CONDA%;%CONDA%\condabin;%CONDA%\Scripts;%CONDA%\Library\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;" + PATH="$PATH:$CONDA:$CONDA\condabin:$CONDA\Scripts:$CONDA\Library\bin:$SystemRoot\system32:$SystemRoot:$SystemRoot\System32\Wbem" + conda update -n base -c defaults conda + conda env update --name base --file environment.yml + pip uninstall -yq mne +elif [ "${TEST_MODE}" == "pip" ]; then + python -m pip install --upgrade pip setuptools + python -m pip install --upgrade numpy scipy vtk + python -m pip install --use-deprecated=legacy-resolver --only-binary="numba,llvmlite" -r requirements.txt +elif [ "${TEST_MODE}" == "pip-pre" ]; then + python -m pip install --upgrade pip setuptools + python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy + python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" scipy pandas scikit-learn h5py Pillow + python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary matplotlib + python -m pip install --upgrade --only-binary vtk vtk; + python -m pip install https://github.com/pyvista/pyvista/zipball/master + python -m pip install https://github.com/pyvista/pyvistaqt/zipball/master + python -m pip install --use-deprecated=legacy-resolver --only-binary="numba,llvmlite" -r requirements.txt +else + echo "Unknown run type ${TEST_MODE}" + exit 1 +fi +python -m pip install -r requirements_testing.txt codecov diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index a54f5dcaabd..e385569a677 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -10,7 +10,8 @@ else # pip 3.9 (missing statsmodels and dipy) pip install --progress-bar off --upgrade --pre --only-binary ":all:" python-dateutil pytz joblib threadpoolctl pip install --use-deprecated=legacy-resolver --progress-bar off --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy scipy pip install --progress-bar off --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" pandas scikit-learn - pip install --progress-bar off --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" matplotlib + # pip install --progress-bar off --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" matplotlib + pip install --progress-bar off --upgrade --pre --only-binary ":all:" matplotlib # built using vtk master branch on an Ubuntu 18.04.5 VM and uploaded to OSF: wget -q https://osf.io/kej3v/download -O vtk-9.0.20201117-cp39-cp39-linux_x86_64.whl pip install vtk-9.0.20201117-cp39-cp39-linux_x86_64.whl From 06fe3dd2b3b61217086936091a1ed822e8489344 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 13:04:25 -0500 Subject: [PATCH 2/8] FIX: Follow order of pip --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a2b271ca67f..498eb79e97e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -198,7 +198,7 @@ stages: TEST_MODE: 'pip' PYTHON_VERSION: '3.7' 3.8 pip pre: - TEST_MODE: 'pre-pip' + TEST_MODE: 'pip-pre' PYTHON_VERSION: '3.8' OPENBLAS_CORETYPE: 'prescott' # workaround for https://github.com/numpy/numpy/issues/16913 steps: @@ -207,7 +207,7 @@ stages: versionSpec: $(PYTHON_VERSION) architecture: $(PYTHON_ARCH) addToPath: true - condition: in(variables['TEST_MODE'], 'pip', 'pre-pip') + condition: in(variables['TEST_MODE'], 'pip', 'pip-pre') displayName: 'Get Python' # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/anaconda # https://github.com/MicrosoftDocs/pipelines-anaconda From bc2c53957c465e642726c709585b625cdbeb792f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 13:05:30 -0500 Subject: [PATCH 3/8] FIX: Better naming --- .github/workflows/linux_pip.yml | 4 ++-- tools/github_actions_dependencies.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_pip.yml b/.github/workflows/linux_pip.yml index d401782e577..e60d21c2728 100644 --- a/.github/workflows/linux_pip.yml +++ b/.github/workflows/linux_pip.yml @@ -1,4 +1,4 @@ -name: 'linux / pip' +name: 'linux / pip-pre' on: push: branches: @@ -22,7 +22,7 @@ jobs: commit-filter: '[skip ci];[ci skip];[skip github]' commit-filter-separator: ';' - # PIP + non-default stim channel + log level info + # PIP-pre + non-default stim channel + log level info job: needs: check_skip if: ${{ needs.check_skip.outputs.skip == 'false' }} diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index e385569a677..d000c941819 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -4,7 +4,7 @@ if [ ! -z "$CONDA_ENV" ]; then pip uninstall -yq mne elif [ ! -z "$CONDA_DEPENDENCIES" ]; then conda install -y $CONDA_DEPENDENCIES -else # pip 3.9 (missing statsmodels and dipy) +else # pip --pre 3.9 (missing statsmodels and dipy) python -m pip install --progress-bar off --upgrade "pip!=20.3.0" setuptools wheel pip uninstall -yq numpy pip install --progress-bar off --upgrade --pre --only-binary ":all:" python-dateutil pytz joblib threadpoolctl From d42421d06b35ee8deed949aa9912bffd5f09c970 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 13:11:54 -0500 Subject: [PATCH 4/8] FIX: Missed one --- tools/azure_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index 056ba4e9507..c6419013d52 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -14,7 +14,7 @@ elif [ "${TEST_MODE}" == "pip-pre" ]; then python -m pip install --upgrade pip setuptools python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" scipy pandas scikit-learn h5py Pillow - python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary matplotlib + python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" matplotlib python -m pip install --upgrade --only-binary vtk vtk; python -m pip install https://github.com/pyvista/pyvista/zipball/master python -m pip install https://github.com/pyvista/pyvistaqt/zipball/master From 37ab2ae3be6914a6a2aee09f319eb8d27161b09a Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 13:15:14 -0500 Subject: [PATCH 5/8] FIX: Export --- tools/azure_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index c6419013d52..16c26e5b480 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -2,7 +2,7 @@ if [ "${TEST_MODE}" == "conda" ]; then echo "##vso[task.prependpath]%CONDA%;%CONDA%\condabin;%CONDA%\Scripts;%CONDA%\Library\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;" - PATH="$PATH:$CONDA:$CONDA\condabin:$CONDA\Scripts:$CONDA\Library\bin:$SystemRoot\system32:$SystemRoot:$SystemRoot\System32\Wbem" + export PATH="$CONDA:$CONDA\condabin:$CONDA\Scripts:$CONDA\Library\bin:$SystemRoot\system32:$SystemRoot:$SystemRoot\System32\Wbem:$PATH" conda update -n base -c defaults conda conda env update --name base --file environment.yml pip uninstall -yq mne From a8c2bf2e5d25e78642db0cbb16f08c5c6d970759 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 13:19:16 -0500 Subject: [PATCH 6/8] FIX: Move back --- azure-pipelines.yml | 3 +++ tools/azure_dependencies.sh | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 498eb79e97e..1e260d4cd48 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -213,6 +213,9 @@ stages: # https://github.com/MicrosoftDocs/pipelines-anaconda # Qt 5.9 post-link failure: # https://github.com/ContinuumIO/anaconda-issues/issues/10949 + - script: echo "##vso[task.prependpath]%CONDA%;%CONDA%\condabin;%CONDA%\Scripts;%CONDA%\Library\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;" + condition: in(variables['TEST_MODE'], 'conda') + displayName: Add conda to PATH, deal with Qt 5.9 bug - bash: | set -e git clone --depth 1 git://github.com/pyvista/gl-ci-helpers.git diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index 16c26e5b480..f1038901141 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -1,8 +1,6 @@ #!/bin/bash -ef if [ "${TEST_MODE}" == "conda" ]; then - echo "##vso[task.prependpath]%CONDA%;%CONDA%\condabin;%CONDA%\Scripts;%CONDA%\Library\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;" - export PATH="$CONDA:$CONDA\condabin:$CONDA\Scripts:$CONDA\Library\bin:$SystemRoot\system32:$SystemRoot:$SystemRoot\System32\Wbem:$PATH" conda update -n base -c defaults conda conda env update --name base --file environment.yml pip uninstall -yq mne From cc70823ca0cc9849c02a23882209a655aa0fad91 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 13:35:02 -0500 Subject: [PATCH 7/8] FIX: To script --- .circleci/config.yml | 9 +-------- tools/circleci_dependencies.sh | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100755 tools/circleci_dependencies.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 671f5fe839c..00282a645e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,14 +87,7 @@ jobs: - run: name: Get Python running command: | - python -m pip uninstall -y pydata-sphinx-theme - python -m pip install --user --upgrade --progress-bar off pip setuptools - python -m pip install --user --upgrade --progress-bar off --pre sphinx - python -m pip install --user --upgrade --progress-bar off -r requirements.txt -r requirements_testing.txt -r requirements_doc.txt - python -m pip install --user --progress-bar off https://github.com/pyvista/pyvista/zipball/master - python -m pip install --user --progress-bar off https://github.com/pyvista/pyvistaqt/zipball/master - python -m pip uninstall -yq pysurfer mayavi - python -m pip install --user -e . + ./tools/circleci_dependencies.sh - save_cache: key: pip-cache diff --git a/tools/circleci_dependencies.sh b/tools/circleci_dependencies.sh new file mode 100755 index 00000000000..0f6dd37907d --- /dev/null +++ b/tools/circleci_dependencies.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ef + +python -m pip uninstall -y pydata-sphinx-theme +python -m pip install --user --upgrade --progress-bar off pip setuptools +python -m pip install --user --upgrade --progress-bar off --pre sphinx +python -m pip install --user --upgrade --progress-bar off -r requirements.txt -r requirements_testing.txt -r requirements_doc.txt +python -m pip install --user --progress-bar off https://github.com/pyvista/pyvista/zipball/master +python -m pip install --user --progress-bar off https://github.com/pyvista/pyvistaqt/zipball/master +python -m pip uninstall -yq pysurfer mayavi +python -m pip install --user -e . From 167197f6aacf5ddc882a023fc83c029ee618b5c9 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 1 Mar 2021 14:20:15 -0500 Subject: [PATCH 8/8] FIX: RC1 --- tools/azure_dependencies.sh | 2 +- tools/github_actions_dependencies.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index f1038901141..eca4e32b8d5 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -12,7 +12,7 @@ elif [ "${TEST_MODE}" == "pip-pre" ]; then python -m pip install --upgrade pip setuptools python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" scipy pandas scikit-learn h5py Pillow - python -m pip install --use-deprecated=legacy-resolver --upgrade --pre --only-binary ":all:" matplotlib + python -m pip install --use-deprecated=legacy-resolver --upgrade --only-binary ":all:" matplotlib python -m pip install --upgrade --only-binary vtk vtk; python -m pip install https://github.com/pyvista/pyvista/zipball/master python -m pip install https://github.com/pyvista/pyvistaqt/zipball/master diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index d000c941819..be2f0fc0503 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -11,7 +11,7 @@ else # pip --pre 3.9 (missing statsmodels and dipy) pip install --use-deprecated=legacy-resolver --progress-bar off --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy scipy pip install --progress-bar off --upgrade --pre --only-binary ":all:" -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" pandas scikit-learn # pip install --progress-bar off --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" matplotlib - pip install --progress-bar off --upgrade --pre --only-binary ":all:" matplotlib + pip install --progress-bar off --upgrade --only-binary ":all:" matplotlib # built using vtk master branch on an Ubuntu 18.04.5 VM and uploaded to OSF: wget -q https://osf.io/kej3v/download -O vtk-9.0.20201117-cp39-cp39-linux_x86_64.whl pip install vtk-9.0.20201117-cp39-cp39-linux_x86_64.whl