Nightly integration tests

.github/workflows/nightly_tests.yml

@@ -1,4 +1,4 @@
-name: Nightly integration tests
+name: Nightly tests on main
 
 on:
   schedule:
@@ -9,12 +9,108 @@ env:
   HF_HOME: /mnt/cache
   OMP_NUM_THREADS: 8
   MKL_NUM_THREADS: 8
-  PYTEST_TIMEOUT: 1000
+  PYTEST_TIMEOUT: 600
   RUN_SLOW: yes
+  RUN_NIGHTLY: yes
 
 jobs:
-  run_slow_tests_apple_m1:
-    name: Slow PyTorch MPS tests on MacOS
+  run_nightly_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: Nightly PyTorch CUDA tests on Ubuntu
+            framework: pytorch
+            runner: docker-gpu
+            image: diffusers/diffusers-pytorch-cuda
+            report: torch_cuda
+          - name: Nightly Flax TPU tests on Ubuntu
+            framework: flax
+            runner: docker-tpu
+            image: diffusers/diffusers-flax-tpu
+            report: flax_tpu
+          - name: Nightly ONNXRuntime CUDA tests on Ubuntu
+            framework: onnxruntime
+            runner: docker-gpu
+            image: diffusers/diffusers-onnxruntime-cuda
+            report: onnx_cuda
+
+    name: ${{ matrix.config.name }}
+
+    runs-on: ${{ matrix.config.runner }}
+
+    container:
+      image: ${{ matrix.config.image }}
+      options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ ${{ matrix.config.runner == 'docker-tpu' && '--privileged' || '--gpus 0'}}
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout diffusers
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: NVIDIA-SMI
+        if: ${{ matrix.config.runner == 'docker-gpu' }}
+        run: |
+          nvidia-smi
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -e .[quality,test]
+          python -m pip install git+https://github.com/huggingface/accelerate
+          python -m pip install -U git+https://github.com/huggingface/transformers
+
+      - name: Environment
+        run: |
+          python utils/print_env.py
+
+      - name: Run nightly PyTorch CUDA tests
+        if: ${{ matrix.config.framework == 'pytorch' }}
+        env:
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+        run: |
+          python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
+            -s -v -k "not Flax and not Onnx" \
+            --make-reports=tests_${{ matrix.config.report }} \
+            tests/
+
+      - name: Run nightly Flax TPU tests
+        if: ${{ matrix.config.framework == 'flax' }}
+        env:
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+        run: |
+          python -m pytest -n 0 \
+            -s -v -k "Flax" \
+            --make-reports=tests_${{ matrix.config.report }} \
+            tests/
+
+      - name: Run nightly ONNXRuntime CUDA tests
+        if: ${{ matrix.config.framework == 'onnxruntime' }}
+        env:
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+        run: |
+          python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
+            -s -v -k "Onnx" \
+            --make-reports=tests_${{ matrix.config.report }} \
+            tests/
+
+      - name: Failure short reports
+        if: ${{ failure() }}
+        run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt
+
+      - name: Test suite reports artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.config.report }}_test_reports
+          path: reports
+
+  run_nightly_tests_apple_m1:
+    name: Nightly PyTorch MPS tests on MacOS
     runs-on: [ self-hosted, apple-m1 ]
 
     steps:
@@ -46,7 +142,7 @@ jobs:
         run: |
           ${CONDA_RUN} python utils/print_env.py
 
-      - name: Run slow PyTorch tests on M1 (MPS)
+      - name: Run nightly PyTorch tests on M1 (MPS)
         shell: arch -arch arm64 bash {0}
         env:
           HF_HOME: /System/Volumes/Data/mnt/cache

.github/workflows/pr_tests.yml

@@ -1,4 +1,4 @@
-name: Run fast tests
+name: Fast tests for PRs
 
 on:
   pull_request:

.github/workflows/push_tests.yml

@@ -1,4 +1,4 @@
-name: Run all tests
+name: Slow tests on main
 
 on:
   push:
@@ -10,7 +10,7 @@ env:
   HF_HOME: /mnt/cache
   OMP_NUM_THREADS: 8
   MKL_NUM_THREADS: 8
-  PYTEST_TIMEOUT: 1000
+  PYTEST_TIMEOUT: 600
   RUN_SLOW: yes
 
 jobs:

src/diffusers/utils/__init__.py

@@ -55,6 +55,7 @@
         load_hf_numpy,
         load_image,
         load_numpy,
+        nightly,
         parse_flag_from_env,
         require_torch_gpu,
         slow,

src/diffusers/utils/testing_utils.py

@@ -83,6 +83,7 @@ def parse_flag_from_env(key, default=False):
 
 
 _run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False)
+_run_nightly_tests = parse_flag_from_env("RUN_NIGHTLY", default=False)
 
 
 def floats_tensor(shape, scale=1.0, rng=None, name=None):
@@ -111,6 +112,16 @@ def slow(test_case):
     return unittest.skipUnless(_run_slow_tests, "test is slow")(test_case)
 
 
+def nightly(test_case):
+    """
+    Decorator marking a test that runs nightly in the diffusers CI.
+
+    Slow tests are skipped by default. Set the RUN_NIGHTLY environment variable to a truthy value to run them.
+
+    """
+    return unittest.skipUnless(_run_nightly_tests, "test is nightly")(test_case)
+
+
 def require_torch(test_case):
     """
     Decorator marking a test that requires PyTorch. These tests are skipped when PyTorch isn't installed.

tests/pipelines/repaint/test_repaint.py

@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import gc
 import unittest
 
 import numpy as np
 import torch
 
 from diffusers import RePaintPipeline, RePaintScheduler, UNet2DModel
-from diffusers.utils.testing_utils import load_image, load_numpy, require_torch_gpu, slow, torch_device
+from diffusers.utils.testing_utils import load_image, load_numpy, nightly, require_torch_gpu, torch_device
 
 from ...test_pipelines_common import PipelineTesterMixin
 
@@ -83,9 +84,14 @@ def test_repaint(self):
         assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3
 
 
-@slow
+@nightly
 @require_torch_gpu
-class RepaintPipelineIntegrationTests(unittest.TestCase):
+class RepaintPipelineNightlyTests(unittest.TestCase):
+    def tearDown(self):
+        super().tearDown()
+        gc.collect()
+        torch.cuda.empty_cache()
+
     def test_celebahq(self):
         original_image = load_image(
             "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/"
@@ -104,6 +110,8 @@ def test_celebahq(self):
         scheduler = RePaintScheduler.from_pretrained(model_id)
 
         repaint = RePaintPipeline(unet=unet, scheduler=scheduler).to(torch_device)
+        repaint.set_progress_bar_config(disable=None)
+        repaint.enable_attention_slicing()
 
         generator = torch.Generator(device=torch_device).manual_seed(0)
         output = repaint(

tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py

@@ -27,7 +27,7 @@
     OnnxStableDiffusionPipeline,
     PNDMScheduler,
 )
-from diffusers.utils.testing_utils import is_onnx_available, require_onnxruntime, require_torch_gpu, slow
+from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu
 
 from ...test_pipelines_onnx_common import OnnxPipelineTesterMixin
 
@@ -128,7 +128,7 @@ def test_pipeline_dpm_multistep(self):
         assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
 
 
-@slow
+@nightly
 @require_onnxruntime
 @require_torch_gpu
 class OnnxStableDiffusionPipelineIntegrationTests(unittest.TestCase):

tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py

@@ -27,7 +27,13 @@
     PNDMScheduler,
 )
 from diffusers.utils import floats_tensor
-from diffusers.utils.testing_utils import is_onnx_available, load_image, require_onnxruntime, require_torch_gpu, slow
+from diffusers.utils.testing_utils import (
+    is_onnx_available,
+    load_image,
+    nightly,
+    require_onnxruntime,
+    require_torch_gpu,
+)
 
 from ...test_pipelines_onnx_common import OnnxPipelineTesterMixin
 
@@ -134,7 +140,7 @@ def test_pipeline_dpm_multistep(self):
         assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1
 
 
-@slow
+@nightly
 @require_onnxruntime
 @require_torch_gpu
 class OnnxStableDiffusionImg2ImgPipelineIntegrationTests(unittest.TestCase):

tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py

@@ -18,7 +18,13 @@
 import numpy as np
 
 from diffusers import LMSDiscreteScheduler, OnnxStableDiffusionInpaintPipeline
-from diffusers.utils.testing_utils import is_onnx_available, load_image, require_onnxruntime, require_torch_gpu, slow
+from diffusers.utils.testing_utils import (
+    is_onnx_available,
+    load_image,
+    nightly,
+    require_onnxruntime,
+    require_torch_gpu,
+)
 
 from ...test_pipelines_onnx_common import OnnxPipelineTesterMixin
 
@@ -32,7 +38,7 @@ class OnnxStableDiffusionPipelineFastTests(OnnxPipelineTesterMixin, unittest.Tes
     pass
 
 
-@slow
+@nightly
 @require_onnxruntime
 @require_torch_gpu
 class OnnxStableDiffusionInpaintPipelineIntegrationTests(unittest.TestCase):

tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint_legacy.py

@@ -22,17 +22,17 @@
     is_onnx_available,
     load_image,
     load_numpy,
+    nightly,
     require_onnxruntime,
     require_torch_gpu,
-    slow,
 )
 
 
 if is_onnx_available():
     import onnxruntime as ort
 
 
-@slow
+@nightly
 @require_onnxruntime
 @require_torch_gpu
 class StableDiffusionOnnxInpaintLegacyPipelineIntegrationTests(unittest.TestCase):