From f51ccccafbfb40d18a60adc9dfb7c754862a84af Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Fri, 14 Oct 2022 16:14:32 -0300 Subject: [PATCH 01/10] add method to enable cuda with minimal gpu usage to stable diffusion --- .../pipeline_stable_diffusion.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 8ae51999a7b3..e4ed312c4c4b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -3,6 +3,7 @@ import torch +from diffusers.utils import is_accelerate_available from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer from ...configuration_utils import FrozenDict @@ -118,6 +119,19 @@ def disable_attention_slicing(self): # set slice_size = `None` to disable `attention slicing` self.enable_attention_slicing(None) + def cuda_with_minimal_gpu_usage(self): + if is_accelerate_available(): + from accelerate.hooks import attach_execution_device_hook + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + self.unet.half().cuda() + attach_execution_device_hook(self.unet, self.unet.device) + self.enable_attention_slicing(1) + + for cpu_offloaded_model in [self.text_encoder, self.vae, self.safety_checker]: + attach_execution_device_hook(cpu_offloaded_model, "cpu") + @torch.no_grad() def __call__( self, @@ -292,7 +306,7 @@ def __call__( self.device ) else: - latents = torch.randn(latents_shape, generator=generator, device=self.device, dtype=latents_dtype) + latents = torch.randn(latents_shape, generator=generator, device=self.unet.device, dtype=latents_dtype) else: if latents.shape != latents_shape: raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") From 88acaf36fddeabc4646972b6cacd4e5fee08c33c Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Fri, 14 Oct 2022 16:56:18 -0300 Subject: [PATCH 02/10] add test to minimal cuda memory usage --- .../pipeline_stable_diffusion.py | 7 +++++-- tests/test_pipelines.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index e4ed312c4c4b..343a20e1f799 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -125,8 +125,11 @@ def cuda_with_minimal_gpu_usage(self): else: raise ImportError("Please install accelerate via `pip install accelerate`") - self.unet.half().cuda() - attach_execution_device_hook(self.unet, self.unet.device) + device = torch.device("cuda") + + self.unet.half().to(device) + attach_execution_device_hook(self.unet, device) + self.unet.forward = torch.autocast("cuda")(self.unet.forward) self.enable_attention_slicing(1) for cpu_offloaded_model in [self.text_encoder, self.vae, self.safety_checker]: diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index a666996e11ff..40b17cd38129 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -2275,3 +2275,23 @@ def test_stable_diffusion_accelerate_load_reduces_memory_footprint(self): tracemalloc.stop() assert peak_accelerate < peak_normal + + @slow + @unittest.skipIf(torch_device == "cpu", "This test is supposed to run on GPU") + def test_stable_diffusion_pipeline_with_unet_on_gpu_only(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + + pipeline_id = "CompVis/stable-diffusion-v1-4" + prompt = "Andromeda galaxy in a bottle" + + pipeline = StableDiffusionPipeline.from_pretrained( + pipeline_id, revision="fp16", torch_dtype=torch.float32, use_auth_token=True + ) + pipeline.cuda_with_minimal_gpu_usage() + + _ = pipeline(prompt) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.2 GB is allocated + assert mem_bytes < 2.2 * 10**9 From a065897e5d8125fb012bc90a5d9b6d6d4d3b1e36 Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Mon, 17 Oct 2022 18:56:29 -0300 Subject: [PATCH 03/10] ensure all models but unet are onn torch.float32 --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 343a20e1f799..682feb7610cf 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -133,6 +133,7 @@ def cuda_with_minimal_gpu_usage(self): self.enable_attention_slicing(1) for cpu_offloaded_model in [self.text_encoder, self.vae, self.safety_checker]: + cpu_offloaded_model.to(torch.float32) attach_execution_device_hook(cpu_offloaded_model, "cpu") @torch.no_grad() From 965dfe100570df86c6bf98d31519160263b39e0c Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 14:43:51 -0300 Subject: [PATCH 04/10] move to cpu_offload along with minor internal changes to make it work --- setup.py | 2 +- src/diffusers/pipeline_utils.py | 2 ++ .../stable_diffusion/pipeline_stable_diffusion.py | 13 ++++--------- tests/test_pipelines.py | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 25e434de2801..16ccf62a98e9 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ # 2. once modified, run: `make deps_table_update` to update src/diffusers/dependency_versions_table.py _deps = [ "Pillow<10.0", # keep the PIL.Image.Resampling deprecation away - "accelerate>=0.11.0", + "accelerate>=0.14.0", "black==22.8", "datasets", "filelock", diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py index 5b1364195c6c..e25193ab772b 100644 --- a/src/diffusers/pipeline_utils.py +++ b/src/diffusers/pipeline_utils.py @@ -206,6 +206,8 @@ def device(self) -> torch.device: for name in module_names.keys(): module = getattr(self, name) if isinstance(module, torch.nn.Module): + if module.device == torch.device("meta"): + return torch.device("cpu") return module.device return torch.device("cpu") diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 682feb7610cf..2167081897db 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -121,20 +121,15 @@ def disable_attention_slicing(self): def cuda_with_minimal_gpu_usage(self): if is_accelerate_available(): - from accelerate.hooks import attach_execution_device_hook + from accelerate import cpu_offload else: raise ImportError("Please install accelerate via `pip install accelerate`") device = torch.device("cuda") - - self.unet.half().to(device) - attach_execution_device_hook(self.unet, device) - self.unet.forward = torch.autocast("cuda")(self.unet.forward) self.enable_attention_slicing(1) - for cpu_offloaded_model in [self.text_encoder, self.vae, self.safety_checker]: - cpu_offloaded_model.to(torch.float32) - attach_execution_device_hook(cpu_offloaded_model, "cpu") + for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae, self.safety_checker]: + cpu_offload(cpu_offloaded_model, device) @torch.no_grad() def __call__( @@ -310,7 +305,7 @@ def __call__( self.device ) else: - latents = torch.randn(latents_shape, generator=generator, device=self.unet.device, dtype=latents_dtype) + latents = torch.randn(latents_shape, generator=generator, device=self.device, dtype=latents_dtype) else: if latents.shape != latents_shape: raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 40b17cd38129..c1052efb6234 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -2293,5 +2293,5 @@ def test_stable_diffusion_pipeline_with_unet_on_gpu_only(self): _ = pipeline(prompt) mem_bytes = torch.cuda.max_memory_allocated() - # make sure that less than 2.2 GB is allocated - assert mem_bytes < 2.2 * 10**9 + # make sure that less than 0.8 GB is allocated + assert mem_bytes < 0.8 * 10**9 From c7b36465fde450d912ba7b17f3e49d91a0e475b4 Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 15:00:27 -0300 Subject: [PATCH 05/10] make it test against accelerate master branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e2c63bf11e4c..2daeb72aab43 100644 --- a/setup.py +++ b/setup.py @@ -178,7 +178,7 @@ def run(self): extras["docs"] = deps_list("hf-doc-builder") extras["training"] = deps_list("accelerate", "datasets", "tensorboard", "modelcards") extras["test"] = deps_list( - "accelerate", + "git+https://github.com/huggingface/accelerate.git", "datasets", "onnxruntime", "pytest", From 45a61d8c0858ccf9e822c5bfcf9f8b1aa8be565d Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 15:11:21 -0300 Subject: [PATCH 06/10] coming back, its official: I don't know how to make it test againt the master branch from accelerate --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2daeb72aab43..e2c63bf11e4c 100644 --- a/setup.py +++ b/setup.py @@ -178,7 +178,7 @@ def run(self): extras["docs"] = deps_list("hf-doc-builder") extras["training"] = deps_list("accelerate", "datasets", "tensorboard", "modelcards") extras["test"] = deps_list( - "git+https://github.com/huggingface/accelerate.git", + "accelerate", "datasets", "onnxruntime", "pytest", From 3af822b918a3682c6843d9804b5696ff742f9dbc Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 15:17:54 -0300 Subject: [PATCH 07/10] make it install accelerate from master on tests --- .github/workflows/pr_tests.yml | 148 +++++++++++++++--------------- .github/workflows/push_tests.yml | 152 +++++++++++++++---------------- 2 files changed, 151 insertions(+), 149 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 493ff51484b0..ea0007804735 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -18,86 +18,88 @@ env: jobs: run_tests_cpu: name: CPU tests on Ubuntu - runs-on: [ self-hosted, docker-gpu ] + runs-on: [self-hosted, docker-gpu] container: image: python:3.7 options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu - python -m pip install -e .[quality,test] - - - name: Environment - run: | - python utils/print_env.py - - - name: Run all fast tests on CPU - run: | - python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/tests_torch_cpu_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: pr_torch_cpu_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu + python -m pip install -e .[quality,test] + python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + run: | + python utils/print_env.py + + - name: Run all fast tests on CPU + run: | + python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_torch_cpu_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: pr_torch_cpu_test_reports + path: reports run_tests_apple_m1: name: MPS tests on Apple M1 - runs-on: [ self-hosted, apple-m1 ] + runs-on: [self-hosted, apple-m1] steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Clean checkout - shell: arch -arch arm64 bash {0} - run: | - git clean -fxd - - - name: Setup miniconda - uses: ./.github/actions/setup-miniconda - with: - python-version: 3.9 - - - name: Install dependencies - shell: arch -arch arm64 bash {0} - run: | - ${CONDA_RUN} python -m pip install --upgrade pip - ${CONDA_RUN} python -m pip install -e .[quality,test] - ${CONDA_RUN} python -m pip install --pre torch==${MPS_TORCH_VERSION} --extra-index-url https://download.pytorch.org/whl/test/cpu - - - name: Environment - shell: arch -arch arm64 bash {0} - run: | - ${CONDA_RUN} python utils/print_env.py - - - name: Run all fast tests on MPS - shell: arch -arch arm64 bash {0} - run: | - ${CONDA_RUN} python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_mps tests/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/tests_torch_mps_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: pr_torch_mps_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Clean checkout + shell: arch -arch arm64 bash {0} + run: | + git clean -fxd + + - name: Setup miniconda + uses: ./.github/actions/setup-miniconda + with: + python-version: 3.9 + + - name: Install dependencies + shell: arch -arch arm64 bash {0} + run: | + ${CONDA_RUN} python -m pip install --upgrade pip + ${CONDA_RUN} python -m pip install -e .[quality,test] + ${CONDA_RUN} python -m pip install --pre torch==${MPS_TORCH_VERSION} --extra-index-url https://download.pytorch.org/whl/test/cpu + ${CONDA_RUN} python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + shell: arch -arch arm64 bash {0} + run: | + ${CONDA_RUN} python utils/print_env.py + + - name: Run all fast tests on MPS + shell: arch -arch arm64 bash {0} + run: | + ${CONDA_RUN} python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_mps tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_torch_mps_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: pr_torch_mps_test_reports + path: reports diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 3e4a81c91c01..976e3c777716 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -15,92 +15,92 @@ env: jobs: run_tests_single_gpu: name: Diffusers tests - runs-on: [ self-hosted, docker-gpu, single-gpu ] + runs-on: [self-hosted, docker-gpu, single-gpu] container: image: nvcr.io/nvidia/pytorch:22.07-py3 options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: NVIDIA-SMI - run: | - nvidia-smi - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip uninstall -y torch torchvision torchtext - python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 - python -m pip install -e .[quality,test] - - - name: Environment - run: | - python utils/print_env.py - - - name: Run all (incl. slow) tests on GPU - env: - HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_gpu tests/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/tests_torch_gpu_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: torch_test_reports - path: reports - - + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: NVIDIA-SMI + run: | + nvidia-smi + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip uninstall -y torch torchvision torchtext + python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 + python -m pip install -e .[quality,test] + python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + run: | + python utils/print_env.py + + - name: Run all (incl. slow) tests on GPU + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_gpu tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_torch_gpu_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: torch_test_reports + path: reports run_examples_single_gpu: name: Examples tests - runs-on: [ self-hosted, docker-gpu, single-gpu ] + runs-on: [self-hosted, docker-gpu, single-gpu] container: image: nvcr.io/nvidia/pytorch:22.07-py3 options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: NVIDIA-SMI - run: | - nvidia-smi - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip uninstall -y torch torchvision torchtext - python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 - python -m pip install -e .[quality,test,training] - - - name: Environment - run: | - python utils/print_env.py - - - name: Run example tests on GPU - env: - HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_gpu examples/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/examples_torch_gpu_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: examples_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: NVIDIA-SMI + run: | + nvidia-smi + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip uninstall -y torch torchvision torchtext + python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 + python -m pip install -e .[quality,test,training] + python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + run: | + python utils/print_env.py + + - name: Run example tests on GPU + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_gpu examples/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/examples_torch_gpu_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: examples_test_reports + path: reports From 358f59a1c80d3d34fa5a6d09d875a93450c4ca6d Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 15:18:42 -0300 Subject: [PATCH 08/10] go back to accelerate>=0.11 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e2c63bf11e4c..7532feb5a360 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ # 2. once modified, run: `make deps_table_update` to update src/diffusers/dependency_versions_table.py _deps = [ "Pillow<10.0", # keep the PIL.Image.Resampling deprecation away - "accelerate>=0.14.0", + "accelerate>=0.11.0", "black==22.8", "datasets", "filelock", From 9793bc843cd393e868183927d365085bcdd08f83 Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 15:59:36 -0300 Subject: [PATCH 09/10] undo prettier formatting on yml files --- .github/workflows/pr_tests.yml | 148 +++++++++++++++--------------- .github/workflows/push_tests.yml | 152 +++++++++++++++---------------- 2 files changed, 150 insertions(+), 150 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index ea0007804735..7ff884fe06d4 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -24,82 +24,82 @@ jobs: options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate - - - name: Environment - run: | - python utils/print_env.py - - - name: Run all fast tests on CPU - run: | - python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/tests_torch_cpu_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: pr_torch_cpu_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu + python -m pip install -e .[quality,test] + python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + run: | + python utils/print_env.py + + - name: Run all fast tests on CPU + run: | + python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_torch_cpu_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: pr_torch_cpu_test_reports + path: reports run_tests_apple_m1: name: MPS tests on Apple M1 - runs-on: [self-hosted, apple-m1] + runs-on: [ self-hosted, apple-m1 ] steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Clean checkout - shell: arch -arch arm64 bash {0} - run: | - git clean -fxd - - - name: Setup miniconda - uses: ./.github/actions/setup-miniconda - with: - python-version: 3.9 - - - name: Install dependencies - shell: arch -arch arm64 bash {0} - run: | - ${CONDA_RUN} python -m pip install --upgrade pip - ${CONDA_RUN} python -m pip install -e .[quality,test] - ${CONDA_RUN} python -m pip install --pre torch==${MPS_TORCH_VERSION} --extra-index-url https://download.pytorch.org/whl/test/cpu - ${CONDA_RUN} python -m pip install git+https://github.com/huggingface/accelerate - - - name: Environment - shell: arch -arch arm64 bash {0} - run: | - ${CONDA_RUN} python utils/print_env.py - - - name: Run all fast tests on MPS - shell: arch -arch arm64 bash {0} - run: | - ${CONDA_RUN} python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_mps tests/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/tests_torch_mps_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: pr_torch_mps_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Clean checkout + shell: arch -arch arm64 bash {0} + run: | + git clean -fxd + + - name: Setup miniconda + uses: ./.github/actions/setup-miniconda + with: + python-version: 3.9 + + - name: Install dependencies + shell: arch -arch arm64 bash {0} + run: | + ${CONDA_RUN} python -m pip install --upgrade pip + ${CONDA_RUN} python -m pip install -e .[quality,test] + ${CONDA_RUN} python -m pip install --pre torch==${MPS_TORCH_VERSION} --extra-index-url https://download.pytorch.org/whl/test/cpu + ${CONDA_RUN} python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + shell: arch -arch arm64 bash {0} + run: | + ${CONDA_RUN} python utils/print_env.py + + - name: Run all fast tests on MPS + shell: arch -arch arm64 bash {0} + run: | + ${CONDA_RUN} python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_mps tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_torch_mps_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: pr_torch_mps_test_reports + path: reports diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 976e3c777716..dfd83aa9af46 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -15,92 +15,92 @@ env: jobs: run_tests_single_gpu: name: Diffusers tests - runs-on: [self-hosted, docker-gpu, single-gpu] + runs-on: [ self-hosted, docker-gpu, single-gpu ] container: image: nvcr.io/nvidia/pytorch:22.07-py3 options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: NVIDIA-SMI - run: | - nvidia-smi - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip uninstall -y torch torchvision torchtext - python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 - python -m pip install -e .[quality,test] - python -m pip install git+https://github.com/huggingface/accelerate - - - name: Environment - run: | - python utils/print_env.py - - - name: Run all (incl. slow) tests on GPU - env: - HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_gpu tests/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/tests_torch_gpu_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: torch_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: NVIDIA-SMI + run: | + nvidia-smi + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip uninstall -y torch torchvision torchtext + python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 + python -m pip install -e .[quality,test] + python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + run: | + python utils/print_env.py + + - name: Run all (incl. slow) tests on GPU + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_gpu tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_torch_gpu_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: torch_test_reports + path: reports run_examples_single_gpu: name: Examples tests - runs-on: [self-hosted, docker-gpu, single-gpu] + runs-on: [ self-hosted, docker-gpu, single-gpu ] container: image: nvcr.io/nvidia/pytorch:22.07-py3 options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache steps: - - name: Checkout diffusers - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: NVIDIA-SMI - run: | - nvidia-smi - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip uninstall -y torch torchvision torchtext - python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 - python -m pip install -e .[quality,test,training] - python -m pip install git+https://github.com/huggingface/accelerate - - - name: Environment - run: | - python utils/print_env.py - - - name: Run example tests on GPU - env: - HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_gpu examples/ - - - name: Failure short reports - if: ${{ failure() }} - run: cat reports/examples_torch_gpu_failures_short.txt - - - name: Test suite reports artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: examples_test_reports - path: reports + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: NVIDIA-SMI + run: | + nvidia-smi + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip uninstall -y torch torchvision torchtext + python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 + python -m pip install -e .[quality,test,training] + python -m pip install git+https://github.com/huggingface/accelerate + + - name: Environment + run: | + python utils/print_env.py + + - name: Run example tests on GPU + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_gpu examples/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/examples_torch_gpu_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: examples_test_reports + path: reports From d1f00bad3c9dca48b30970d107af406902f12cbe Mon Sep 17 00:00:00 2001 From: Pi Esposito Date: Thu, 20 Oct 2022 16:00:48 -0300 Subject: [PATCH 10/10] undo prettier formatting on yml files againn --- .github/workflows/pr_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 7ff884fe06d4..5798d059e328 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -18,7 +18,7 @@ env: jobs: run_tests_cpu: name: CPU tests on Ubuntu - runs-on: [self-hosted, docker-gpu] + runs-on: [ self-hosted, docker-gpu ] container: image: python:3.7 options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/