From f30d9fcae245bab3772c7ea9ff4d3622f4c3ee39 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 12:25:02 -0400 Subject: [PATCH 01/14] Add CuPy dependency Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 268dfb87d7..9e03935242 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,3 +32,4 @@ sphinx-autodoc-typehints==1.11.1 sphinx-rtd-theme==0.5.2 cucim~=0.19.0; platform_system == "Linux" openslide-python==1.1.2 +cupy==8.6.0 From be0d7a536fae366cf3791ffc7a16ceb74f75e650 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 12:25:04 -0400 Subject: [PATCH 02/14] Remove CUDA-based dependencies (only cupy for now) Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 8f7d9c72c2..6f279cd278 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -77,6 +77,9 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }} + - name: Remove CUDA-required dependencies + run: | + python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines([t for t in txt if not t.startswith('cupy')]); f.close()" - if: runner.os == 'windows' name: Install torch cpu from pytorch.org (Windows only) run: | From 6c1feb40d05600e7ffb4ec8f14142b5fd481cb72 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 12:36:08 -0400 Subject: [PATCH 03/14] Exclude CUDA-based dependecies from flake8-py3 Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 6f279cd278..62b4fafc01 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -30,6 +30,9 @@ jobs: with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} + - name: Remove CUDA-required dependencies + run: | + python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines([t for t in txt if not t.startswith('cupy')]); f.close()" - name: Install dependencies run: | python -m pip install --upgrade pip wheel From e8dd7e93dc2fae6cab398aca27df746a158ea9fa Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 13:47:04 -0400 Subject: [PATCH 04/14] Install CuPy binary specific to each CUDA version Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 5 +++-- requirements-dev.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 62b4fafc01..f8eaf5e669 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -32,7 +32,7 @@ jobs: key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} - name: Remove CUDA-required dependencies run: | - python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines([t for t in txt if not t.startswith('cupy')]); f.close()" + sed -i '/^cupy/d' requirements-dev.txt - name: Install dependencies run: | python -m pip install --upgrade pip wheel @@ -82,7 +82,7 @@ jobs: key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }} - name: Remove CUDA-required dependencies run: | - python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines([t for t in txt if not t.startswith('cupy')]); f.close()" + sed -i '/^cupy/d' requirements-dev.txt - if: runner.os == 'windows' name: Install torch cpu from pytorch.org (Windows only) run: | @@ -286,6 +286,7 @@ jobs: which python python -m pip install --upgrade pip wheel python -m pip install ${{ matrix.pytorch }} + sed -i 's/cupy=/cupy-${{ matrix.environment }:-7}=/g' requirements-dev.txt python -m pip install -r requirements-dev.txt python -m pip list - name: Run quick tests (GPU) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9e03935242..e07f39d982 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,4 +32,4 @@ sphinx-autodoc-typehints==1.11.1 sphinx-rtd-theme==0.5.2 cucim~=0.19.0; platform_system == "Linux" openslide-python==1.1.2 -cupy==8.6.0 +cupy==9.0.0 From f988530cdbb972bcff2a168e3c7069f14ce8c99a Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 14:59:42 -0400 Subject: [PATCH 05/14] Make cupy package name lowercase Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index f8eaf5e669..5aef3f1d8d 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -286,7 +286,7 @@ jobs: which python python -m pip install --upgrade pip wheel python -m pip install ${{ matrix.pytorch }} - sed -i 's/cupy=/cupy-${{ matrix.environment }:-7}=/g' requirements-dev.txt + sed -i 's/cupy=/cupy-cuda${{ matrix.environment }: -3}=/g' requirements-dev.txt python -m pip install -r requirements-dev.txt python -m pip list - name: Run quick tests (GPU) From f50778346a014de25a46c131cb13f6ed5c027a7e Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 19:45:05 -0400 Subject: [PATCH 06/14] Correct matrix environment substring extraction Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 5aef3f1d8d..e28a0929ed 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -286,7 +286,8 @@ jobs: which python python -m pip install --upgrade pip wheel python -m pip install ${{ matrix.pytorch }} - sed -i 's/cupy=/cupy-cuda${{ matrix.environment }: -3}=/g' requirements-dev.txt + matenv=${{ matrix.environment }} + sed -i 's/cupy=/cupy-cuda${matenv: -3}=/g' requirements-dev.txt python -m pip install -r requirements-dev.txt python -m pip list - name: Run quick tests (GPU) From 8fb305bfed7d564a2e3e5b80849370ffa8771bce Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 20:51:39 -0400 Subject: [PATCH 07/14] Make sed work on Mac too Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index e28a0929ed..8232145e74 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -32,7 +32,7 @@ jobs: key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} - name: Remove CUDA-required dependencies run: | - sed -i '/^cupy/d' requirements-dev.txt + sed -i.bak -e "/^cupy/d" requirements-dev.txt - name: Install dependencies run: | python -m pip install --upgrade pip wheel @@ -82,7 +82,7 @@ jobs: key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }} - name: Remove CUDA-required dependencies run: | - sed -i '/^cupy/d' requirements-dev.txt + sed -i.bak -e "/^cupy/d" requirements-dev.txt - if: runner.os == 'windows' name: Install torch cpu from pytorch.org (Windows only) run: | @@ -287,7 +287,7 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install ${{ matrix.pytorch }} matenv=${{ matrix.environment }} - sed -i 's/cupy=/cupy-cuda${matenv: -3}=/g' requirements-dev.txt + sed -i.bak -e "s/cupy=/cupy-cuda${matenv: -3}=/g" requirements-dev.txt python -m pip install -r requirements-dev.txt python -m pip list - name: Run quick tests (GPU) From 1d81bcbb1b1791bf00fd5b9f4114cb9dc48b9fff Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 21:24:11 -0400 Subject: [PATCH 08/14] Exclude windows from cupy logic Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 8232145e74..fa1824a252 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -80,7 +80,8 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }} - - name: Remove CUDA-required dependencies + - if: runner.os != 'windows' + name: Remove CUDA-required dependencies (Mac and Linux) run: | sed -i.bak -e "/^cupy/d" requirements-dev.txt - if: runner.os == 'windows' From 89446c4670b5790ec3c56dabc1a289a1ae22575d Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 21:25:27 -0400 Subject: [PATCH 09/14] Revert to CuPy 8.6.0 due to availability on PyPI Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e07f39d982..9e03935242 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,4 +32,4 @@ sphinx-autodoc-typehints==1.11.1 sphinx-rtd-theme==0.5.2 cucim~=0.19.0; platform_system == "Linux" openslide-python==1.1.2 -cupy==9.0.0 +cupy==8.6.0 From 8c0bc6de426cdb793d89d3e23fdc9243431080ca Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 3 May 2021 23:36:28 -0400 Subject: [PATCH 10/14] Separate creating CuPy binary dependency from installng dependencies Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index fa1824a252..3cdd20fc1d 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -282,13 +282,16 @@ jobs: python get-pip.py && \ rm get-pip.py; fi + - name: Create the CUDA-dependent binary requirements for the environment + run: | + matenv=${{ matrix.environment }} + sed -i.bak -e "s/cupy=/cupy-cuda${matenv: -3}=/g" requirements-dev.txt + cat "requirements-dev.txt" - name: Install dependencies run: | which python python -m pip install --upgrade pip wheel python -m pip install ${{ matrix.pytorch }} - matenv=${{ matrix.environment }} - sed -i.bak -e "s/cupy=/cupy-cuda${matenv: -3}=/g" requirements-dev.txt python -m pip install -r requirements-dev.txt python -m pip list - name: Run quick tests (GPU) From f1856f0f05a55e28558e6e2b79112f43e92ebc12 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Tue, 4 May 2021 00:09:10 -0400 Subject: [PATCH 11/14] Use sed to extract CUDA version Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 3cdd20fc1d..adb833ae67 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -284,8 +284,7 @@ jobs: fi - name: Create the CUDA-dependent binary requirements for the environment run: | - matenv=${{ matrix.environment }} - sed -i.bak -e "s/cupy=/cupy-cuda${matenv: -3}=/g" requirements-dev.txt + sed -i.bak -e "s/cupy=/cupy-cuda$(echo ${{ matrix.pytorch }} | sed 's/^.*CUDA//')=/g" requirements-dev.txt cat "requirements-dev.txt" - name: Install dependencies run: | From a8413bf69beb8ed91bdcbebc078635faaa4c5b56 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Tue, 4 May 2021 00:11:28 -0400 Subject: [PATCH 12/14] Fix a typo Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index adb833ae67..f12e71273c 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -284,7 +284,7 @@ jobs: fi - name: Create the CUDA-dependent binary requirements for the environment run: | - sed -i.bak -e "s/cupy=/cupy-cuda$(echo ${{ matrix.pytorch }} | sed 's/^.*CUDA//')=/g" requirements-dev.txt + sed -i.bak -e "s/cupy=/cupy-cuda$(echo ${{ matrix.environment }} | sed 's/^.*CUDA//')=/g" requirements-dev.txt cat "requirements-dev.txt" - name: Install dependencies run: | From a2feac32e2da0b1d255a88ce853ba94d8073958f Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Tue, 4 May 2021 07:11:57 -0400 Subject: [PATCH 13/14] Remove cucim mac only workaround Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index f12e71273c..4ce068dd50 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -90,10 +90,6 @@ jobs: python -m pip install torch==1.8.1+cpu torchvision==0.9.1+cpu -f https://download.pytorch.org/whl/torch_stable.html # min. requirements for windows instances python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines(txt[1:12]); f.close()" - - if: runner.os == 'macos' - name: Remove cucim installation (Mac only) - run: | - python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines([t for t in txt if not t.startswith('cucim')]); f.close()" - name: Install the dependencies run: | python -m pip install torch==1.8.1 torchvision==0.9.1 From c71698d644a7e5080c09f5f7e12b9dbd3fcd97e6 Mon Sep 17 00:00:00 2001 From: Behrooz <3968947+behxyz@users.noreply.github.com> Date: Mon, 26 Jul 2021 15:56:57 +0000 Subject: [PATCH 14/14] Install cupy on only one cuda environment Signed-off-by: Behrooz <3968947+behxyz@users.noreply.github.com> --- .github/workflows/pythonapp.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 4ce068dd50..92a838d0a9 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -30,9 +30,6 @@ jobs: with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} - - name: Remove CUDA-required dependencies - run: | - sed -i.bak -e "/^cupy/d" requirements-dev.txt - name: Install dependencies run: | python -m pip install --upgrade pip wheel @@ -80,10 +77,6 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }} - - if: runner.os != 'windows' - name: Remove CUDA-required dependencies (Mac and Linux) - run: | - sed -i.bak -e "/^cupy/d" requirements-dev.txt - if: runner.os == 'windows' name: Install torch cpu from pytorch.org (Windows only) run: | @@ -278,10 +271,10 @@ jobs: python get-pip.py && \ rm get-pip.py; fi - - name: Create the CUDA-dependent binary requirements for the environment + - if: matrix.environment == "PT18+CUDA112" + name: Add CuPy dependency to the latest CUDA environment run: | - sed -i.bak -e "s/cupy=/cupy-cuda$(echo ${{ matrix.environment }} | sed 's/^.*CUDA//')=/g" requirements-dev.txt - cat "requirements-dev.txt" + echo "cupy-cuda112" >> requirements-dev.txt - name: Install dependencies run: | which python