From e6198d0975522f5f86a754062c247ecbe50bb263 Mon Sep 17 00:00:00 2001 From: anton- Date: Mon, 12 Dec 2022 14:02:54 +0100 Subject: [PATCH 01/22] [WIP] Nightly integration tests --- src/diffusers/utils/testing_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/diffusers/utils/testing_utils.py b/src/diffusers/utils/testing_utils.py index bf398e5b6fe5..3622002889b1 100644 --- a/src/diffusers/utils/testing_utils.py +++ b/src/diffusers/utils/testing_utils.py @@ -83,6 +83,7 @@ def parse_flag_from_env(key, default=False): _run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_nightly_tests = parse_flag_from_env("RUN_NIGHTLY", default=False) def floats_tensor(shape, scale=1.0, rng=None, name=None): @@ -111,6 +112,16 @@ def slow(test_case): return unittest.skipUnless(_run_slow_tests, "test is slow")(test_case) +def nightly(test_case): + """ + Decorator marking a test that runs nightly in the diffusers CI. + + Slow tests are skipped by default. Set the RUN_SLOW environment variable to a truthy value to run them. + + """ + return unittest.skipUnless(_run_nightly_tests, "test is nightly")(test_case) + + def require_torch(test_case): """ Decorator marking a test that requires PyTorch. These tests are skipped when PyTorch isn't installed. From 18b216de6d904cff0e6067c7523f24c5252aceb2 Mon Sep 17 00:00:00 2001 From: anton- Date: Mon, 12 Dec 2022 15:01:41 +0100 Subject: [PATCH 02/22] initial SD tests --- src/diffusers/utils/__init__.py | 1 + .../stable_diffusion/test_stable_diffusion.py | 46 +++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/diffusers/utils/__init__.py b/src/diffusers/utils/__init__.py index 30ead70e43f3..e6d2ec031119 100644 --- a/src/diffusers/utils/__init__.py +++ b/src/diffusers/utils/__init__.py @@ -58,6 +58,7 @@ parse_flag_from_env, require_torch_gpu, slow, + nightly, torch_all_close, torch_device, ) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 4748a41feae7..8db32eab5daf 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -14,6 +14,7 @@ # limitations under the License. import gc +import random import tempfile import time import unittest @@ -32,8 +33,8 @@ UNet2DConditionModel, logging, ) -from diffusers.utils import load_numpy, slow, torch_device -from diffusers.utils.testing_utils import CaptureLogger, require_torch_gpu +from diffusers.utils import load_numpy, slow, nightly, torch_device +from diffusers.utils.testing_utils import CaptureLogger, require_torch_gpu, floats_tensor from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer from ...test_pipelines_common import PipelineTesterMixin @@ -435,7 +436,7 @@ def test_stable_diffusion_height_width_opt(self): @slow @require_torch_gpu -class StableDiffusionPipelineIntegrationTests(unittest.TestCase): +class StableDiffusionPipelineSlowTests(unittest.TestCase): def tearDown(self): # clean up the VRAM after each test super().tearDown() @@ -710,3 +711,42 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.8 GB is allocated assert mem_bytes < 2.8 * 10**9 + + +@nightly +@require_torch_gpu +class StableDiffusionPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + latents = np.random.RandomState(0).standard_normal((1, 4, 64, 64)).astype(np.float32) + latents = torch.from_numpy(latents).to(device) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_1_4_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0856, 0.0690, 0.0358, 0.0493, 0.0607, 0.0496, 0.0517, 0.0446, 0.0346]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 From d8c9a6f7ea6125a55978e154dfe43c36a8d23e11 Mon Sep 17 00:00:00 2001 From: anton- Date: Mon, 12 Dec 2022 17:27:58 +0100 Subject: [PATCH 03/22] update SD slow tests --- .github/workflows/push_tests.yml | 15 +- src/diffusers/utils/__init__.py | 2 +- .../stable_diffusion/test_stable_diffusion.py | 289 ++++++++---------- 3 files changed, 127 insertions(+), 179 deletions(-) diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index ad1821621ffb..83385370dad8 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -4,6 +4,9 @@ on: push: branches: - main + pull_request: + branches: + - main env: DIFFUSERS_IS_CI: yes @@ -24,16 +27,6 @@ jobs: runner: docker-gpu image: diffusers/diffusers-pytorch-cuda report: torch_cuda - - name: Slow Flax TPU tests on Ubuntu - framework: flax - runner: docker-tpu - image: diffusers/diffusers-flax-tpu - report: flax_tpu - - name: Slow ONNXRuntime CUDA tests on Ubuntu - framework: onnxruntime - runner: docker-gpu - image: diffusers/diffusers-onnxruntime-cuda - report: onnx_cuda name: ${{ matrix.config.name }} @@ -74,7 +67,7 @@ jobs: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ - -s -v -k "not Flax and not Onnx" \ + -s -v -k "StableDiffusionPipeline" \ --make-reports=tests_${{ matrix.config.report }} \ tests/ diff --git a/src/diffusers/utils/__init__.py b/src/diffusers/utils/__init__.py index e6d2ec031119..e5a4d323e3eb 100644 --- a/src/diffusers/utils/__init__.py +++ b/src/diffusers/utils/__init__.py @@ -55,10 +55,10 @@ load_hf_numpy, load_image, load_numpy, + nightly, parse_flag_from_env, require_torch_gpu, slow, - nightly, torch_all_close, torch_device, ) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 8db32eab5daf..b6f692553bcb 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -13,8 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. + +import os + + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + import gc -import random import tempfile import time import unittest @@ -33,14 +38,15 @@ UNet2DConditionModel, logging, ) -from diffusers.utils import load_numpy, slow, nightly, torch_device -from diffusers.utils.testing_utils import CaptureLogger, require_torch_gpu, floats_tensor +from diffusers.utils import load_numpy, nightly, slow, torch_device +from diffusers.utils.testing_utils import CaptureLogger, require_torch_gpu from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer from ...test_pipelines_common import PipelineTesterMixin torch.backends.cuda.matmul.allow_tf32 = False +torch.use_deterministic_algorithms(True) class StableDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): @@ -438,125 +444,121 @@ def test_stable_diffusion_height_width_opt(self): @require_torch_gpu class StableDiffusionPipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() - def test_stable_diffusion(self): - # make sure here that pndm scheduler skips prk + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_1_1_pndm(self): sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1") sd_pipe = sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) - prompt = "A painting of a squirrel eating a burger" - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast("cuda"): - output = sd_pipe( - [prompt], generator=generator, guidance_scale=6.0, num_inference_steps=20, output_type="np" - ) - - image = output.images - - image_slice = image[0, -3:, -3:, -1] + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.8887, 0.915, 0.91, 0.894, 0.909, 0.912, 0.919, 0.925, 0.883]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 - - def test_stable_diffusion_fast_ddim(self): - scheduler = DDIMScheduler.from_pretrained("CompVis/stable-diffusion-v1-1", subfolder="scheduler") + expected_slice = np.array([0.43625, 0.43554, 0.36670, 0.40660, 0.39703, 0.38658, 0.43936, 0.43557, 0.40592]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 - sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1", scheduler=scheduler) + def test_stable_diffusion_1_4_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") sd_pipe = sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) - prompt = "A painting of a squirrel eating a burger" - generator = torch.Generator(device=torch_device).manual_seed(0) + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - with torch.autocast("cuda"): - output = sd_pipe([prompt], generator=generator, num_inference_steps=2, output_type="numpy") - image = output.images + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.57400, 0.47841, 0.31625, 0.63583, 0.58306, 0.55056, 0.50825, 0.56306, 0.55748]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 - image_slice = image[0, -3:, -3:, -1] + def test_stable_diffusion_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.9326, 0.923, 0.951, 0.9365, 0.9214, 0.951, 0.9365, 0.9414, 0.918]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + expected_slice = np.array([0.38019, 0.28647, 0.27321, 0.40377, 0.38290, 0.35446, 0.39218, 0.38165, 0.42239]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 - def test_lms_stable_diffusion_pipeline(self): - model_id = "CompVis/stable-diffusion-v1-1" - pipe = StableDiffusionPipeline.from_pretrained(model_id).to(torch_device) - pipe.set_progress_bar_config(disable=None) - scheduler = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe.scheduler = scheduler + def test_stable_diffusion_lms(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) - prompt = "a photograph of an astronaut riding a horse" - generator = torch.Generator(device=torch_device).manual_seed(0) - image = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ).images + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.9077, 0.9254, 0.9181, 0.9227, 0.9213, 0.9367, 0.9399, 0.9406, 0.9024]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + expected_slice = np.array([0.10542, 0.09620, 0.07332, 0.09015, 0.09382, 0.07597, 0.08496, 0.07806, 0.06455]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 - def test_stable_diffusion_memory_chunking(self): + def test_stable_diffusion_attention_slicing(self): torch.cuda.reset_peak_memory_stats() - model_id = "CompVis/stable-diffusion-v1-4" - pipe = StableDiffusionPipeline.from_pretrained(model_id, revision="fp16", torch_dtype=torch.float16) - pipe.to(torch_device) + pipe = StableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) - prompt = "a photograph of an astronaut riding a horse" - - # make attention efficient + # enable attention slicing pipe.enable_attention_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - output_chunked = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image_chunked = output_chunked.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image_sliced = pipe(**inputs).images mem_bytes = torch.cuda.max_memory_allocated() torch.cuda.reset_peak_memory_stats() # make sure that less than 3.75 GB is allocated assert mem_bytes < 3.75 * 10**9 - # disable chunking + # disable slicing pipe.disable_attention_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - output = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image = output.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images # make sure that more than 3.75 GB is allocated mem_bytes = torch.cuda.max_memory_allocated() assert mem_bytes > 3.75 * 10**9 - assert np.abs(image_chunked.flatten() - image.flatten()).max() < 1e-3 + assert np.abs(image_sliced - image).max() < 1e-3 def test_stable_diffusion_vae_slicing(self): torch.cuda.reset_peak_memory_stats() - model_id = "CompVis/stable-diffusion-v1-4" - pipe = StableDiffusionPipeline.from_pretrained(model_id, revision="fp16", torch_dtype=torch.float16) - pipe.to(torch_device) + pipe = StableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "a photograph of an astronaut riding a horse" - # enable vae slicing pipe.enable_vae_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - output_chunked = pipe( - [prompt] * 4, generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image_chunked = output_chunked.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + inputs["prompt"] = [inputs["prompt"]] * 4 + inputs["latents"] = torch.cat([inputs["latents"]] * 4) + image_sliced = pipe(**inputs).images mem_bytes = torch.cuda.max_memory_allocated() torch.cuda.reset_peak_memory_stats() @@ -565,92 +567,58 @@ def test_stable_diffusion_vae_slicing(self): # disable vae slicing pipe.disable_vae_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - output = pipe( - [prompt] * 4, generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image = output.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + inputs["prompt"] = [inputs["prompt"]] * 4 + inputs["latents"] = torch.cat([inputs["latents"]] * 4) + image = pipe(**inputs).images # make sure that more than 4 GB is allocated mem_bytes = torch.cuda.max_memory_allocated() assert mem_bytes > 4e9 # There is a small discrepancy at the image borders vs. a fully batched version. - assert np.abs(image_chunked.flatten() - image.flatten()).max() < 3e-3 + assert np.abs(image_sliced - image).max() < 4e-3 - def test_stable_diffusion_text2img_pipeline_fp16(self): - torch.cuda.reset_peak_memory_stats() - model_id = "CompVis/stable-diffusion-v1-4" - pipe = StableDiffusionPipeline.from_pretrained(model_id, revision="fp16", torch_dtype=torch.float16) + def test_stable_diffusion_fp16_vs_autocast(self): + pipe = StableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16 + ) pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) - prompt = "a photograph of an astronaut riding a horse" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output_chunked = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image_chunked = output_chunked.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image_fp16 = pipe(**inputs).images - generator = torch.Generator(device=torch_device).manual_seed(0) with torch.autocast(torch_device): - output = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image = output.images + inputs = self.get_inputs(torch_device) + image_autocast = pipe(**inputs).images # Make sure results are close enough - diff = np.abs(image_chunked.flatten() - image.flatten()) + diff = np.abs(image_fp16.flatten() - image_autocast.flatten()) # They ARE different since ops are not run always at the same precision # however, they should be extremely close. assert diff.mean() < 2e-2 - def test_stable_diffusion_text2img_pipeline_default(self): - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text2img/astronaut_riding_a_horse.npy" - ) - - model_id = "CompVis/stable-diffusion-v1-4" - pipe = StableDiffusionPipeline.from_pretrained(model_id, safety_checker=None) - pipe.to(torch_device) - pipe.set_progress_bar_config(disable=None) - pipe.enable_attention_slicing() - - prompt = "astronaut riding a horse" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe(prompt=prompt, guidance_scale=7.5, generator=generator, output_type="np") - image = output.images[0] - - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 5e-3 - - def test_stable_diffusion_text2img_intermediate_state(self): + def test_stable_diffusion_intermediate_state(self): number_of_steps = 0 - def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: - test_callback_fn.has_been_called = True + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True nonlocal number_of_steps number_of_steps += 1 - if step == 0: + if step == 1: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array( - [1.8285, 1.2857, -0.1024, 1.2406, -2.3068, 1.0747, -0.0818, -0.6520, -2.9506] - ) + expected_slice = np.array([-0.5713, -0.3018, -0.9814, 0.04663, -0.879, 0.76, -1.734, 0.1044, 1.161]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-3 - elif step == 50: + elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array( - [1.1078, 1.5803, 0.2773, -0.0589, -1.7928, -0.3665, -0.4695, -1.0727, -1.1601] - ) + expected_slice = np.array([-0.1885, -0.3022, -1.012, -0.514, -0.477, 0.6143, -0.9336, 0.6553, 1.453]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-2 - test_callback_fn.has_been_called = False + callback_fn.has_been_called = False pipe = StableDiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16 @@ -659,20 +627,10 @@ def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> No pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "Andromeda galaxy in a bottle" - - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - pipe( - prompt=prompt, - num_inference_steps=50, - guidance_scale=7.5, - generator=generator, - callback=test_callback_fn, - callback_steps=1, - ) - assert test_callback_fn.has_been_called - assert number_of_steps == 50 + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == inputs["num_inference_steps"] def test_stable_diffusion_low_cpu_mem_usage(self): pipeline_id = "CompVis/stable-diffusion-v1-4" @@ -686,7 +644,7 @@ def test_stable_diffusion_low_cpu_mem_usage(self): start_time = time.time() _ = StableDiffusionPipeline.from_pretrained( - pipeline_id, revision="fp16", torch_dtype=torch.float16, use_auth_token=True, low_cpu_mem_usage=False + pipeline_id, revision="fp16", torch_dtype=torch.float16, low_cpu_mem_usage=False ) normal_load_time = time.time() - start_time @@ -697,16 +655,16 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): torch.cuda.reset_max_memory_allocated() torch.cuda.reset_peak_memory_stats() - pipeline_id = "CompVis/stable-diffusion-v1-4" - prompt = "Andromeda galaxy in a bottle" - - pipeline = StableDiffusionPipeline.from_pretrained(pipeline_id, revision="fp16", torch_dtype=torch.float16) - pipeline = pipeline.to(torch_device) - pipeline.enable_attention_slicing(1) - pipeline.enable_sequential_cpu_offload() + pipe = StableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() - generator = torch.Generator(device=torch_device).manual_seed(0) - _ = pipeline(prompt, generator=generator, num_inference_steps=5) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.8 GB is allocated @@ -717,17 +675,13 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): @require_torch_gpu class StableDiffusionPipelineNightlyTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() def get_inputs(self, device, seed=0): - if str(device).startswith("mps"): - generator = torch.manual_seed(seed) - else: - generator = torch.Generator(device=device).manual_seed(seed) - latents = np.random.RandomState(0).standard_normal((1, 4, 64, 64)).astype(np.float32) + generator = torch.Generator(device=device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)).astype(np.float32) latents = torch.from_numpy(latents).to(device) inputs = { "prompt": "a photograph of an astronaut riding a horse", @@ -744,9 +698,10 @@ def test_stable_diffusion_1_4_pndm(self): sd_pipe.set_progress_bar_config(disable=None) inputs = self.get_inputs(torch_device) - image = sd_pipe(**inputs).images - image_slice = image[0, -3:, -3:, -1] + image = sd_pipe(**inputs).images[0] - assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.0856, 0.0690, 0.0358, 0.0493, 0.0607, 0.0496, 0.0517, 0.0446, 0.0346]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_4_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-4) From 8bb254c274bb5e9ce49a39b8a03b2529a52dc759 Mon Sep 17 00:00:00 2001 From: anton- Date: Mon, 12 Dec 2022 17:29:56 +0100 Subject: [PATCH 04/22] style --- tests/pipelines/stable_diffusion/test_stable_diffusion.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index b6f692553bcb..1a8652562218 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -14,12 +14,8 @@ # limitations under the License. -import os - - -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - import gc +import os import tempfile import time import unittest @@ -45,6 +41,7 @@ from ...test_pipelines_common import PipelineTesterMixin +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False torch.use_deterministic_algorithms(True) From c4d215bd29e13cfd35e2563962819e19929a5c0e Mon Sep 17 00:00:00 2001 From: anton- Date: Mon, 12 Dec 2022 18:21:13 +0100 Subject: [PATCH 05/22] repaint --- tests/pipelines/repaint/test_repaint.py | 14 +++++++++----- .../stable_diffusion/test_stable_diffusion.py | 8 ++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/pipelines/repaint/test_repaint.py b/tests/pipelines/repaint/test_repaint.py index d1ecd3c06ee4..8622fe274191 100644 --- a/tests/pipelines/repaint/test_repaint.py +++ b/tests/pipelines/repaint/test_repaint.py @@ -13,21 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import gc import unittest import numpy as np import torch from diffusers import RePaintPipeline, RePaintScheduler, UNet2DModel -from diffusers.utils.testing_utils import load_image, load_numpy, require_torch_gpu, slow, torch_device - +from diffusers.utils.testing_utils import load_image, load_numpy, require_torch_gpu, torch_device, nightly torch.backends.cuda.matmul.allow_tf32 = False - -@slow +@nightly @require_torch_gpu -class RepaintPipelineIntegrationTests(unittest.TestCase): +class RepaintPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + def test_celebahq(self): original_image = load_image( "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/" diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 1a8652562218..f54f74007805 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -676,10 +676,10 @@ def tearDown(self): gc.collect() torch.cuda.empty_cache() - def get_inputs(self, device, seed=0): + def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) - latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)).astype(np.float32) - latents = torch.from_numpy(latents).to(device) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) inputs = { "prompt": "a photograph of an astronaut riding a horse", "latents": latents, @@ -701,4 +701,4 @@ def test_stable_diffusion_1_4_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_4_pndm.npy" ) - assert np.allclose(image, expected_image, atol=1e-4) + assert np.allclose(image, expected_image, atol=1e-3) From da5af01920bd46c09f796e9e5015d241c0fbf9f4 Mon Sep 17 00:00:00 2001 From: anton- Date: Wed, 14 Dec 2022 18:00:16 +0100 Subject: [PATCH 06/22] ImageVariations --- .github/workflows/push_tests.yml | 6 +- tests/pipelines/repaint/test_repaint.py | 4 +- .../stable_diffusion/test_stable_diffusion.py | 58 +++++++++++ .../test_stable_diffusion_image_variation.py | 95 ++++++++++++++----- tests/test_pipelines.py | 13 ++- 5 files changed, 145 insertions(+), 31 deletions(-) diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 83385370dad8..9392a86bb444 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -4,9 +4,9 @@ on: push: branches: - main - pull_request: - branches: - - main +# pull_request: +# branches: +# - main env: DIFFUSERS_IS_CI: yes diff --git a/tests/pipelines/repaint/test_repaint.py b/tests/pipelines/repaint/test_repaint.py index 8622fe274191..0f0e690f5e48 100644 --- a/tests/pipelines/repaint/test_repaint.py +++ b/tests/pipelines/repaint/test_repaint.py @@ -20,10 +20,12 @@ import torch from diffusers import RePaintPipeline, RePaintScheduler, UNet2DModel -from diffusers.utils.testing_utils import load_image, load_numpy, require_torch_gpu, torch_device, nightly +from diffusers.utils.testing_utils import load_image, load_numpy, nightly, require_torch_gpu, torch_device + torch.backends.cuda.matmul.allow_tf32 = False + @nightly @require_torch_gpu class RepaintPipelineNightlyTests(unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index f54f74007805..7d5449bec6ed 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -28,6 +28,7 @@ DDIMScheduler, EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, StableDiffusionPipeline, @@ -702,3 +703,60 @@ def test_stable_diffusion_1_4_pndm(self): ) assert np.allclose(image, expected_image, atol=1e-3) + + def test_stable_diffusion_1_5_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_5_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_stable_diffusion_lms(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_4_lms.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_stable_diffusion_euler(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = EulerDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_euler.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_stable_diffusion_dpm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_dpm_multi.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index 4be95322d6b6..1c619812ce6b 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import gc import random import unittest @@ -27,14 +28,16 @@ StableDiffusionImageVariationPipeline, UNet2DConditionModel, ) -from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device +from diffusers.utils import floats_tensor, load_image, load_numpy, slow, nightly, torch_device from diffusers.utils.testing_utils import require_torch_gpu from transformers import CLIPVisionConfig, CLIPVisionModelWithProjection from ...test_pipelines_common import PipelineTesterMixin +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False +torch.use_deterministic_algorithms(True) class StableDiffusionImageVariationPipelineFastTests(PipelineTesterMixin, unittest.TestCase): @@ -173,43 +176,42 @@ def test_stable_diffusion_img_variation_num_images_per_prompt(self): @slow @require_torch_gpu -class StableDiffusionImageVariationPipelineIntegrationTests(unittest.TestCase): +class StableDiffusionImageVariationPipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() - def test_stable_diffusion_img_variation_pipeline_default(self): + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/vermeer.jpg" ) init_image = init_image.resize((512, 512)) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/vermeer.npy" - ) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "image": init_image, + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs - model_id = "fusing/sd-image-variations-diffusers" - pipe = StableDiffusionImageVariationPipeline.from_pretrained( - model_id, - safety_checker=None, - ) - pipe.to(torch_device) - pipe.set_progress_bar_config(disable=None) - pipe.enable_attention_slicing() + def test_stable_diffusion_img_variation_pipeline_default(self): + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("lambdalabs/sd-image-variations-diffusers") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - init_image, - guidance_scale=7.5, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - assert image.shape == (512, 512, 3) - # img2img is flaky across GPUs even in fp32, so using MAE here - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.43625, 0.43554, 0.36670, 0.40660, 0.39703, 0.38658, 0.43936, 0.43557, 0.40592]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 def test_stable_diffusion_img_variation_intermediate_state(self): number_of_steps = 0 @@ -293,3 +295,44 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.6 GB is allocated assert mem_bytes < 2.6 * 10**9 + +@nightly +@require_torch_gpu +class StableDiffusionImageVariationPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/vermeer.jpg" + ) + init_image = init_image.resize((512, 512)) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "image": init_image, + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_img_variation_pndm(self): + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("fusing/sd-image-variations-diffusers") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/lambdalabs_variations_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) \ No newline at end of file diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 3a6d5139e95b..8826c8e76c3e 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -46,7 +46,7 @@ ) from diffusers.pipeline_utils import DiffusionPipeline from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME -from diffusers.utils import CONFIG_NAME, WEIGHTS_NAME, floats_tensor, slow, torch_device +from diffusers.utils import CONFIG_NAME, WEIGHTS_NAME, floats_tensor, nightly, slow, torch_device from diffusers.utils.testing_utils import CaptureLogger, get_tests_dir, require_torch_gpu from parameterized import parameterized from PIL import Image @@ -648,6 +648,7 @@ def test_optional_components(self): @slow +@require_torch_gpu class PipelineSlowTests(unittest.TestCase): def tearDown(self): # clean up the VRAM after each test @@ -790,6 +791,16 @@ def test_output_format(self): assert isinstance(images, list) assert isinstance(images[0], PIL.Image.Image) + +@nightly +@require_torch_gpu +class PipelineNightlyTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + def test_ddpm_ddim_equality_batched(self): seed = 0 model_id = "google/ddpm-cifar10-32" From d80a93a613fe921a4d899969c1fd3ffabba2aee3 Mon Sep 17 00:00:00 2001 From: anton- Date: Thu, 15 Dec 2022 17:25:26 +0100 Subject: [PATCH 07/22] style --- tests/pipelines/stable_diffusion/test_stable_diffusion.py | 2 +- .../test_stable_diffusion_image_variation.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 7d5449bec6ed..e91ff63715f5 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -26,9 +26,9 @@ from diffusers import ( AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, - DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, StableDiffusionPipeline, diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index 1c619812ce6b..53dace36b3e4 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import gc +import os import random import unittest @@ -28,7 +28,7 @@ StableDiffusionImageVariationPipeline, UNet2DConditionModel, ) -from diffusers.utils import floats_tensor, load_image, load_numpy, slow, nightly, torch_device +from diffusers.utils import floats_tensor, load_image, load_numpy, nightly, slow, torch_device from diffusers.utils.testing_utils import require_torch_gpu from transformers import CLIPVisionConfig, CLIPVisionModelWithProjection @@ -296,6 +296,7 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): # make sure that less than 2.6 GB is allocated assert mem_bytes < 2.6 * 10**9 + @nightly @require_torch_gpu class StableDiffusionImageVariationPipelineNightlyTests(unittest.TestCase): @@ -335,4 +336,4 @@ def test_img_variation_pndm(self): "/stable_diffusion_imgvar/lambdalabs_variations_pndm.npy" ) - assert np.allclose(image, expected_image, atol=1e-3) \ No newline at end of file + assert np.allclose(image, expected_image, atol=1e-3) From 0b976f8586f2c31a081eb7f7957724a2f6de515a Mon Sep 17 00:00:00 2001 From: anton- Date: Thu, 15 Dec 2022 19:59:57 +0100 Subject: [PATCH 08/22] finish imgvar --- .../stable_diffusion/test_stable_diffusion.py | 25 +++++- .../test_stable_diffusion_image_variation.py | 78 +++++++++---------- .../test_stable_diffusion_img2img.py | 68 +++++++++++++++- 3 files changed, 122 insertions(+), 49 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index e91ff63715f5..bdce3b5ee651 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -699,7 +699,8 @@ def test_stable_diffusion_1_4_pndm(self): image = sd_pipe(**inputs).images[0] expected_image = load_numpy( - "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_4_pndm.npy" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_pndm.npy" ) assert np.allclose(image, expected_image, atol=1e-3) @@ -712,7 +713,23 @@ def test_stable_diffusion_1_5_pndm(self): image = sd_pipe(**inputs).images[0] expected_image = load_numpy( - "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_5_pndm.npy" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_5_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_stable_diffusion_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_ddim.npy" ) assert np.allclose(image, expected_image, atol=1e-3) @@ -726,7 +743,8 @@ def test_stable_diffusion_lms(self): image = sd_pipe(**inputs).images[0] expected_image = load_numpy( - "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_text2img/stable_diffusion_1_4_lms.npy" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_lms.npy" ) assert np.allclose(image, expected_image, atol=1e-3) @@ -752,6 +770,7 @@ def test_stable_diffusion_dpm(self): sd_pipe.set_progress_bar_config(disable=None) inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 image = sd_pipe(**inputs).images[0] expected_image = load_numpy( diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index 3e2058bb82ec..b4cd5692d478 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -23,7 +23,7 @@ from diffusers import ( AutoencoderKL, - LMSDiscreteScheduler, + DPMSolverMultistepScheduler, PNDMScheduler, StableDiffusionImageVariationPipeline, UNet2DConditionModel, @@ -189,7 +189,8 @@ def tearDown(self): def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/vermeer.jpg" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/input_image.jpg" ) init_image = init_image.resize((512, 512)) latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) @@ -220,30 +221,24 @@ def test_stable_diffusion_img_variation_pipeline_default(self): def test_stable_diffusion_img_variation_intermediate_state(self): number_of_steps = 0 - def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: - test_callback_fn.has_been_called = True + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True nonlocal number_of_steps number_of_steps += 1 - if step == 0: + if step == 1: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] expected_slice = np.array([1.83, 1.293, -0.09705, 1.256, -2.293, 1.091, -0.0809, -0.65, -2.953]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-3 - elif step == 37: + elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] expected_slice = np.array([2.285, 2.703, 1.969, 0.696, -1.323, 0.9253, -0.5464, -1.521, -2.537]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 - test_callback_fn.has_been_called = False - - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((512, 512)) + callback_fn.has_been_called = False pipe = StableDiffusionImageVariationPipeline.from_pretrained( "fusing/sd-image-variations-diffusers", @@ -253,48 +248,27 @@ def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> No pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - pipe( - init_image, - num_inference_steps=50, - guidance_scale=7.5, - generator=generator, - callback=test_callback_fn, - callback_steps=1, - ) - assert test_callback_fn.has_been_called - assert number_of_steps == 50 + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == inputs["num_inference_steps"] def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() torch.cuda.reset_peak_memory_stats() - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((512, 512)) - model_id = "fusing/sd-image-variations-diffusers" - lms = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") pipe = StableDiffusionImageVariationPipeline.from_pretrained( - model_id, scheduler=lms, safety_checker=None, torch_dtype=torch.float16 + model_id, safety_checker=None, torch_dtype=torch.float16 ) - pipe.to(torch_device) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() - generator = torch.Generator(device=torch_device).manual_seed(0) - _ = pipe( - init_image, - guidance_scale=7.5, - generator=generator, - output_type="np", - num_inference_steps=5, - ) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.6 GB is allocated @@ -312,7 +286,8 @@ def tearDown(self): def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/vermeer.jpg" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/input_image.jpg" ) init_image = init_image.resize((512, 512)) latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) @@ -341,3 +316,20 @@ def test_img_variation_pndm(self): ) assert np.allclose(image, expected_image, atol=1e-3) + + def test_img_variation_dpm(self): + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("fusing/sd-image-variations-diffusers") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/lambdalabs_variations_dpm_multi.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index a9d341a1387a..d9ea5dc7d465 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -14,6 +14,7 @@ # limitations under the License. import gc +import os import random import unittest @@ -28,14 +29,16 @@ StableDiffusionImg2ImgPipeline, UNet2DConditionModel, ) -from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device +from diffusers.utils import floats_tensor, load_image, load_numpy, nightly, slow, torch_device from diffusers.utils.testing_utils import require_torch_gpu from transformers import CLIPImageProcessor, CLIPTextConfig, CLIPTextModel, CLIPTokenizer from ...test_pipelines_common import PipelineTesterMixin +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False +torch.use_deterministic_algorithms(True) class StableDiffusionImg2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): @@ -212,13 +215,30 @@ def test_stable_diffusion_img2img_num_images_per_prompt(self): @slow @require_torch_gpu -class StableDiffusionImg2ImgPipelineIntegrationTests(unittest.TestCase): +class StableDiffusionImg2ImgPipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + init_image = init_image.resize((768, 512)) + inputs = { + "prompt": "a fantasy landscape, concept art, high resolution", + "image": init_image, + "generator": generator, + "num_inference_steps": 50, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + def test_stable_diffusion_img2img_pipeline_default(self): init_image = load_image( "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" @@ -420,3 +440,45 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.2 GB is allocated assert mem_bytes < 2.2 * 10**9 + + +@nightly +@require_torch_gpu +class StableDiffusionImg2ImgPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + init_image = init_image.resize((768, 512)) + inputs = { + "prompt": "a fantasy landscape, concept art, high resolution", + "image": init_image, + "generator": generator, + "num_inference_steps": 50, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_img2img_pndm(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) From edf2d1f53cca8112ec26607d3b654cb990b736a1 Mon Sep 17 00:00:00 2001 From: anton- Date: Thu, 15 Dec 2022 21:39:20 +0100 Subject: [PATCH 09/22] img2img tests --- .github/workflows/nightly_tests.yml | 91 ++++++- .github/workflows/pr_tests.yml | 2 +- .github/workflows/push_tests.yml | 10 +- .../stable_diffusion/test_stable_diffusion.py | 18 +- .../test_stable_diffusion_img2img.py | 248 +++++++----------- 5 files changed, 207 insertions(+), 162 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index c46612bddcf2..a39acd44cb42 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -3,16 +3,105 @@ name: Nightly integration tests on: schedule: - cron: "0 0 * * *" # every day at midnight + pull_request: + branches: + - main env: DIFFUSERS_IS_CI: yes HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 - PYTEST_TIMEOUT: 1000 + PYTEST_TIMEOUT: 600 RUN_SLOW: yes + RUN_NIGHTLY: yes jobs: + run_slow_tests: + strategy: + fail-fast: false + matrix: + config: + - name: Slow PyTorch CUDA tests on Ubuntu + framework: pytorch + runner: docker-gpu + image: diffusers/diffusers-pytorch-cuda + report: torch_cuda + + name: ${{ matrix.config.name }} + + runs-on: ${{ matrix.config.runner }} + + container: + image: ${{ matrix.config.image }} + options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ ${{ matrix.config.runner == 'docker-tpu' && '--privileged' || '--gpus 0'}} + + defaults: + run: + shell: bash + + steps: + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: NVIDIA-SMI + if: ${{ matrix.config.runner == 'docker-gpu' }} + run: | + nvidia-smi + + - name: Install dependencies + run: | + python -m pip install -e .[quality,test] + python -m pip install git+https://github.com/huggingface/accelerate + python -m pip install -U git+https://github.com/huggingface/transformers + + - name: Environment + run: | + python utils/print_env.py + + - name: Run slow PyTorch CUDA tests + if: ${{ matrix.config.framework == 'pytorch' }} + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + -s -v -k "StableDiffusionPipeline" \ + --make-reports=tests_${{ matrix.config.report }} \ + tests/ + + - name: Run slow Flax TPU tests + if: ${{ matrix.config.framework == 'flax' }} + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 0 \ + -s -v -k "Flax" \ + --make-reports=tests_${{ matrix.config.report }} \ + tests/ + + - name: Run slow ONNXRuntime CUDA tests + if: ${{ matrix.config.framework == 'onnxruntime' }} + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + -s -v -k "Onnx" \ + --make-reports=tests_${{ matrix.config.report }} \ + tests/ + + - name: Failure short reports + if: ${{ failure() }} + run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.config.report }}_test_reports + path: reports + run_slow_tests_apple_m1: name: Slow PyTorch MPS tests on MacOS runs-on: [ self-hosted, apple-m1 ] diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 10670f33cf2f..0763538c7bd1 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -1,4 +1,4 @@ -name: Run fast tests +name: Fast tests for PRs on: pull_request: diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 9392a86bb444..e1541d4f43ce 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -1,19 +1,19 @@ -name: Run all tests +name: Slow tests on main on: push: branches: - main -# pull_request: -# branches: -# - main + pull_request: + branches: + - main env: DIFFUSERS_IS_CI: yes HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 - PYTEST_TIMEOUT: 1000 + PYTEST_TIMEOUT: 600 RUN_SLOW: yes jobs: diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index bdce3b5ee651..a5d423987f6e 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -487,7 +487,7 @@ def test_stable_diffusion_1_4_pndm(self): assert np.abs(image_slice - expected_slice).max() < 1e-4 def test_stable_diffusion_ddim(self): - sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) sd_pipe = sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) @@ -501,7 +501,7 @@ def test_stable_diffusion_ddim(self): assert np.abs(image_slice - expected_slice).max() < 1e-4 def test_stable_diffusion_lms(self): - sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) sd_pipe = sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) @@ -514,6 +514,20 @@ def test_stable_diffusion_lms(self): expected_slice = np.array([0.10542, 0.09620, 0.07332, 0.09015, 0.09382, 0.07597, 0.08496, 0.07806, 0.06455]) assert np.abs(image_slice - expected_slice).max() < 1e-4 + def test_stable_diffusion_dpm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.03503, 0.03494, 0.01087, 0.03128, 0.02552, 0.00803, 0.00742, 0.00372, 0.00000]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 + def test_stable_diffusion_attention_slicing(self): torch.cuda.reset_peak_memory_stats() pipe = StableDiffusionPipeline.from_pretrained( diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index d9ea5dc7d465..96d259cb34db 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -224,218 +224,114 @@ def tearDown(self): def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/sketch-mountains-input.jpg" ) init_image = init_image.resize((768, 512)) inputs = { "prompt": "a fantasy landscape, concept art, high resolution", "image": init_image, "generator": generator, - "num_inference_steps": 50, + "num_inference_steps": 3, "strength": 0.75, "guidance_scale": 7.5, "output_type": "numpy", } return inputs - def test_stable_diffusion_img2img_pipeline_default(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((768, 512)) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape.npy" - ) - - model_id = "CompVis/stable-diffusion-v1-4" - pipe = StableDiffusionImg2ImgPipeline.from_pretrained( - model_id, - safety_checker=None, - ) + def test_stable_diffusion_img2img_default(self): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "A fantasy landscape, trending on artstation" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - assert image.shape == (512, 768, 3) - # img2img is flaky across GPUs even in fp32, so using MAE here - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.27150, 0.14849, 0.15605, 0.26740, 0.16954, 0.18204, 0.31470, 0.26311, 0.24525]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 - def test_stable_diffusion_img2img_pipeline_k_lms(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((768, 512)) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape_k_lms.npy" - ) - - model_id = "CompVis/stable-diffusion-v1-4" - lms = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionImg2ImgPipeline.from_pretrained( - model_id, - scheduler=lms, - safety_checker=None, - ) + def test_stable_diffusion_img2img_k_lms(self): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "A fantasy landscape, trending on artstation" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - assert image.shape == (512, 768, 3) - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.04890, 0.04862, 0.06422, 0.04655, 0.05108, 0.05307, 0.05926, 0.08759, 0.06852]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 - def test_stable_diffusion_img2img_pipeline_ddim(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((768, 512)) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape_ddim.npy" - ) - - model_id = "CompVis/stable-diffusion-v1-4" - ddim = DDIMScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionImg2ImgPipeline.from_pretrained( - model_id, - scheduler=ddim, - safety_checker=None, - ) + def test_stable_diffusion_img2img_ddim(self): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "A fantasy landscape, trending on artstation" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - assert image.shape == (512, 768, 3) - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.06069, 0.05703, 0.08054, 0.05797, 0.06286, 0.06234, 0.08438, 0.11151, 0.08068]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 def test_stable_diffusion_img2img_intermediate_state(self): number_of_steps = 0 - def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: - test_callback_fn.has_been_called = True + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True nonlocal number_of_steps number_of_steps += 1 - if step == 0: + if step == 1: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 96) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([0.9052, -0.0184, 0.4810, 0.2898, 0.5851, 1.4920, 0.5362, 1.9838, 0.0530]) + expected_slice = np.array([0.7705, 0.1045, 0.5, 3.393, 3.723, 4.273, 2.467, 3.486, 1.758]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 - elif step == 37: + elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 96) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([0.7071, 0.7831, 0.8300, 1.8140, 1.7840, 1.9402, 1.3651, 1.6590, 1.2828]) - assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-2 + expected_slice = np.array([0.765, 0.1047, 0.4973, 3.375, 3.709, 4.258, 2.451, 3.46, 1.755]) + assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 - test_callback_fn.has_been_called = False - - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((768, 512)) + callback_fn.has_been_called = False pipe = StableDiffusionImg2ImgPipeline.from_pretrained( - "CompVis/stable-diffusion-v1-4", - revision="fp16", - torch_dtype=torch.float16, + "CompVis/stable-diffusion-v1-4", safety_checker=None, revision="fp16", torch_dtype=torch.float16 ) - pipe.to(torch_device) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "A fantasy landscape, trending on artstation" - - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - pipe( - prompt=prompt, - image=init_image, - strength=0.75, - num_inference_steps=50, - guidance_scale=7.5, - generator=generator, - callback=test_callback_fn, - callback_steps=1, - ) - assert test_callback_fn.has_been_called - assert number_of_steps == 37 + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 2 def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() torch.cuda.reset_peak_memory_stats() - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" - ) - init_image = init_image.resize((768, 512)) - - model_id = "CompVis/stable-diffusion-v1-4" - lms = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") pipe = StableDiffusionImg2ImgPipeline.from_pretrained( - model_id, scheduler=lms, safety_checker=None, device_map="auto", revision="fp16", torch_dtype=torch.float16 + "CompVis/stable-diffusion-v1-4", safety_checker=None, revision="fp16", torch_dtype=torch.float16 ) - pipe.to(torch_device) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() - prompt = "A fantasy landscape, trending on artstation" - - generator = torch.Generator(device=torch_device).manual_seed(0) - _ = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - num_inference_steps=5, - ) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.2 GB is allocated @@ -453,8 +349,8 @@ def tearDown(self): def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/img2img/sketch-mountains-input.jpg" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/sketch-mountains-input.jpg" ) init_image = init_image.resize((768, 512)) inputs = { @@ -482,3 +378,49 @@ def test_img2img_pndm(self): ) assert np.allclose(image, expected_image, atol=1e-3) + + def test_img2img_ddim(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_ddim.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_img2img_lms(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_lms.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_img2img_dpm(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 30 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_dpm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) From 55f10e40af6f4da2b2ee0e22508e52010c696bc9 Mon Sep 17 00:00:00 2001 From: anton- Date: Thu, 15 Dec 2022 21:40:48 +0100 Subject: [PATCH 10/22] debug --- .github/workflows/nightly_tests.yml | 2 +- .github/workflows/push_tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index a39acd44cb42..ef22441a2c86 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -67,7 +67,7 @@ jobs: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ - -s -v -k "StableDiffusionPipeline" \ + -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ tests/ diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index e1541d4f43ce..de2e0d555003 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -67,7 +67,7 @@ jobs: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ - -s -v -k "StableDiffusionPipeline" \ + -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ tests/ From 0fb4bf2ea4f75d8c1814bf4a3aba1f665977a481 Mon Sep 17 00:00:00 2001 From: anton- Date: Thu, 15 Dec 2022 22:37:30 +0100 Subject: [PATCH 11/22] inpaint 1.5 --- .github/workflows/nightly_tests.yml | 8 +- .github/workflows/push_tests.yml | 6 +- .../test_stable_diffusion_inpaint.py | 302 +++++++++--------- 3 files changed, 162 insertions(+), 154 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index ef22441a2c86..a4f65f916bd5 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -1,11 +1,11 @@ -name: Nightly integration tests +name: Nightly tests on main on: schedule: - cron: "0 0 * * *" # every day at midnight - pull_request: - branches: - - main +# pull_request: +# branches: +# - main env: DIFFUSERS_IS_CI: yes diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index de2e0d555003..288ed1fe3f9b 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -4,9 +4,9 @@ on: push: branches: - main - pull_request: - branches: - - main +# pull_request: +# branches: +# - main env: DIFFUSERS_IS_CI: yes diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index cc5749855701..e46ebeba889f 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -14,6 +14,7 @@ # limitations under the License. import gc +import os import random import unittest @@ -22,13 +23,14 @@ from diffusers import ( AutoencoderKL, + DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, StableDiffusionInpaintPipeline, UNet2DConditionModel, ) from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint import prepare_mask_and_masked_image -from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device +from diffusers.utils import floats_tensor, load_image, load_numpy, nightly, slow, torch_device from diffusers.utils.testing_utils import require_torch_gpu from PIL import Image from transformers import CLIPImageProcessor, CLIPTextConfig, CLIPTextModel, CLIPTokenizer @@ -36,7 +38,9 @@ from ...test_pipelines_common import PipelineTesterMixin +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False +torch.use_deterministic_algorithms(True) class StableDiffusionInpaintPipelineFastTests(PipelineTesterMixin, unittest.TestCase): @@ -163,208 +167,212 @@ def test_stable_diffusion_inpaint_with_num_images_per_prompt(self): @slow @require_torch_gpu -class StableDiffusionInpaintPipelineIntegrationTests(unittest.TestCase): +class StableDiffusionInpaintPipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() - def test_stable_diffusion_inpaint_pipeline(self): + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" ) mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" - ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint" - "/yellow_cat_sitting_on_a_park_bench.npy" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" ) + inputs = { + "prompt": "Face of a yellow cat, high resolution, sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs - model_id = "runwayml/stable-diffusion-inpainting" - pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None) + def test_stable_diffusion_inpaint_ddim(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.05978, 0.10983, 0.10514, 0.07922, 0.08483, 0.08587, 0.05302, 0.03218, 0.01636]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - generator=generator, - output_type="np", + def test_stable_diffusion_inpaint_fp16(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", revision="fp16", torch_dtype=torch.float16, safety_checker=None ) - image = output.images[0] + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 1e-3 + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - def test_stable_diffusion_inpaint_pipeline_fp16(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" - ) - mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" - ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint" - "/yellow_cat_sitting_on_a_park_bench_fp16.npy" - ) + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.06152, 0.11060, 0.10449, 0.07959, 0.08643, 0.08496, 0.05420, 0.03247, 0.01831]) + assert np.abs(expected_slice - image_slice).max() < 1e-2 - model_id = "runwayml/stable-diffusion-inpainting" + def test_stable_diffusion_inpaint_pndm(self): pipe = StableDiffusionInpaintPipeline.from_pretrained( - model_id, - revision="fp16", - torch_dtype=torch.float16, - safety_checker=None, + "runwayml/stable-diffusion-inpainting", safety_checker=None ) + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "Face of a yellow cat, high resolution, sitting on a park bench" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 5e-1 + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.06892, 0.06994, 0.07905, 0.05366, 0.04709, 0.04890, 0.04107, 0.05083, 0.04180]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 - def test_stable_diffusion_inpaint_pipeline_pndm(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" - ) - mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" - ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint" - "/yellow_cat_sitting_on_a_park_bench_pndm.npy" + def test_stable_diffusion_inpaint_k_lms(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None ) - - model_id = "runwayml/stable-diffusion-inpainting" - pndm = PNDMScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None, scheduler=pndm) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - generator=generator, - output_type="np", + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.23513, 0.22413, 0.29442, 0.24243, 0.26214, 0.30329, 0.26431, 0.25025, 0.25197]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 + + def test_stable_diffusion_inpaint_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None, revision="fp16", torch_dtype=torch.float16 ) - image = output.images[0] + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.2 GB is allocated + assert mem_bytes < 2.2 * 10**9 - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 1e-2 - def test_stable_diffusion_inpaint_pipeline_k_lms(self): +@nightly +@require_torch_gpu +class StableDiffusionInpaintPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" ) mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" ) + inputs = { + "prompt": "Face of a yellow cat, high resolution, sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_inpaint_ddim(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint" - "/yellow_cat_sitting_on_a_park_bench_k_lms.npy" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_ddim.npy" ) - model_id = "runwayml/stable-diffusion-inpainting" - pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None) - pipe.to(torch_device) + assert np.allclose(image, expected_image, atol=1e-3) - # switch to LMS - pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) - - pipe.set_progress_bar_config(disable=None) - pipe.enable_attention_slicing() + def test_inpaint_pndm(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.scheduler = PNDMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) - prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - generator=generator, - output_type="np", + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_pndm.npy" ) - image = output.images[0] - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 1e-2 + assert np.allclose(image, expected_image, atol=1e-3) - @unittest.skipIf(torch_device == "cpu", "This test is supposed to run on GPU") - def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): - torch.cuda.empty_cache() - torch.cuda.reset_max_memory_allocated() - torch.cuda.reset_peak_memory_stats() + def test_inpaint_lms(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" - ) - mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" - ) + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] - model_id = "runwayml/stable-diffusion-inpainting" - pndm = PNDMScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionInpaintPipeline.from_pretrained( - model_id, - safety_checker=None, - scheduler=pndm, - device_map="auto", - revision="fp16", - torch_dtype=torch.float16, + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_lms.npy" ) - pipe.to(torch_device) - pipe.set_progress_bar_config(disable=None) - pipe.enable_attention_slicing(1) - pipe.enable_sequential_cpu_offload() - prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + assert np.allclose(image, expected_image, atol=1e-3) + + def test_inpaint_dpm(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) - generator = torch.Generator(device=torch_device).manual_seed(0) - _ = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - generator=generator, - num_inference_steps=5, - output_type="np", + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 30 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_dpm_multi.npy" ) - mem_bytes = torch.cuda.max_memory_allocated() - # make sure that less than 2.2 GB is allocated - assert mem_bytes < 2.2 * 10**9 + assert np.allclose(image, expected_image, atol=1e-3) class StableDiffusionInpaintingPrepareMaskAndMaskedImageTests(unittest.TestCase): From 817f1c9f7e8028dc99311df545b670582d952591 Mon Sep 17 00:00:00 2001 From: anton- Date: Thu, 15 Dec 2022 23:05:58 +0100 Subject: [PATCH 12/22] inpaint legacy --- .../test_stable_diffusion_img2img.py | 4 + .../test_stable_diffusion_inpaint_legacy.py | 247 +++++++++++------- 2 files changed, 151 insertions(+), 100 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index 96d259cb34db..4c6eb9bf1236 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -24,6 +24,7 @@ from diffusers import ( AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, StableDiffusionImg2ImgPipeline, @@ -381,6 +382,7 @@ def test_img2img_pndm(self): def test_img2img_ddim(self): sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) @@ -396,6 +398,7 @@ def test_img2img_ddim(self): def test_img2img_lms(self): sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) @@ -411,6 +414,7 @@ def test_img2img_lms(self): def test_img2img_dpm(self): sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py index 9207fa7fd4e4..da589f9ba94e 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py @@ -22,15 +22,16 @@ from diffusers import ( AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, - StableDiffusionInpaintPipeline, StableDiffusionInpaintPipelineLegacy, UNet2DConditionModel, UNet2DModel, VQModel, ) -from diffusers.utils import floats_tensor, load_image, slow, torch_device +from diffusers.utils import floats_tensor, load_image, nightly, slow, torch_device from diffusers.utils.testing_utils import load_numpy, require_torch_gpu from PIL import Image from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer @@ -340,146 +341,192 @@ def test_stable_diffusion_inpaint_legacy_num_images_per_prompt(self): @slow @require_torch_gpu -class StableDiffusionInpaintLegacyPipelineIntegrationTests(unittest.TestCase): +class StableDiffusionInpaintLegacyPipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() - def test_stable_diffusion_inpaint_legacy_pipeline(self): + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" ) mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint" - "/red_cat_sitting_on_a_park_bench.npy" + inputs = { + "prompt": "A red cat sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_inpaint_legacy_pndm(self): + pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained( + "CompVis/stable-diffusion-v1-4", safety_checker=None ) - - model_id = "CompVis/stable-diffusion-v1-4" - pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "A red cat sitting on a park bench" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.27200, 0.29103, 0.34405, 0.21418, 0.26317, 0.34281, 0.18033, 0.24911, 0.32028]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 - def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" - ) - mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" - ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint" - "/red_cat_sitting_on_a_park_bench_k_lms.npy" - ) - - model_id = "CompVis/stable-diffusion-v1-4" - lms = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionInpaintPipeline.from_pretrained( - model_id, - scheduler=lms, - safety_checker=None, + def test_stable_diffusion_inpaint_legacy_k_lms(self): + pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained( + "CompVis/stable-diffusion-v1-4", safety_checker=None ) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "A red cat sitting on a park bench" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - ) - image = output.images[0] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.29014, 0.28882, 0.32835, 0.26502, 0.28182, 0.31162, 0.29297, 0.29534, 0.28214]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 def test_stable_diffusion_inpaint_legacy_intermediate_state(self): number_of_steps = 0 - def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: - test_callback_fn.has_been_called = True + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True nonlocal number_of_steps number_of_steps += 1 - if step == 0: + if step == 1: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array( - [-0.5472, 1.1218, -0.5505, -0.9390, -1.0794, 0.4063, 0.5158, 0.6429, -1.5246] - ) + expected_slice = np.array([-0.103, 1.415, -0.02197, -0.5107, -0.5903, 0.1953, 0.75, 0.3477, -1.356]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 - elif step == 37: + elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([0.4781, 1.1572, 0.6258, 0.2291, 0.2554, -0.1443, 0.7085, -0.1598, -0.5659]) + expected_slice = np.array([0.4802, 1.154, 0.628, 0.2319, 0.2593, -0.1455, 0.7075, -0.1617, -0.5615]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 - test_callback_fn.has_been_called = False + callback_fn.has_been_called = False + + pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained( + "CompVis/stable-diffusion-v1-4", safety_checker=None, revision="fp16", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 2 + + +@nightly +@require_torch_gpu +class StableDiffusionInpaintLegacyPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" ) mask_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" ) + inputs = { + "prompt": "A red cat sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 50, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_inpaint_pndm(self): + sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] - pipe = StableDiffusionInpaintPipeline.from_pretrained( - "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16 + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_inpaint_ddim(self): + sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_ddim.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_inpaint_lms(self): + sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_lms.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_inpaint_dpm(self): + sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 30 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_dpm_multi.npy" ) - pipe.to(torch_device) - pipe.set_progress_bar_config(disable=None) - pipe.enable_attention_slicing() - prompt = "A red cat sitting on a park bench" - - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - pipe( - prompt=prompt, - image=init_image, - mask_image=mask_image, - strength=0.75, - num_inference_steps=50, - guidance_scale=7.5, - generator=generator, - callback=test_callback_fn, - callback_steps=1, - ) - assert test_callback_fn.has_been_called - assert number_of_steps == 37 + assert np.allclose(image, expected_image, atol=1e-3) From 0214037434256fea872afaca7ccf4f93656be8c9 Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 09:33:32 +0100 Subject: [PATCH 13/22] torch isn't happy about deterministic ops --- .../stable_diffusion/test_stable_diffusion.py | 3 - .../test_stable_diffusion_image_variation.py | 3 - .../test_stable_diffusion_img2img.py | 3 - .../test_stable_diffusion_inpaint.py | 6 +- .../test_stable_diffusion_depth.py | 270 ++++++++++-------- 5 files changed, 154 insertions(+), 131 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index a5d423987f6e..a5e7f8ecf3f4 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -15,7 +15,6 @@ import gc -import os import tempfile import time import unittest @@ -42,9 +41,7 @@ from ...test_pipelines_common import PipelineTesterMixin -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False -torch.use_deterministic_algorithms(True) class StableDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index b4cd5692d478..7824689da0dd 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -14,7 +14,6 @@ # limitations under the License. import gc -import os import random import unittest @@ -36,9 +35,7 @@ from ...test_pipelines_common import PipelineTesterMixin -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False -torch.use_deterministic_algorithms(True) class StableDiffusionImageVariationPipelineFastTests(PipelineTesterMixin, unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index 4c6eb9bf1236..cbc94451ae27 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -14,7 +14,6 @@ # limitations under the License. import gc -import os import random import unittest @@ -37,9 +36,7 @@ from ...test_pipelines_common import PipelineTesterMixin -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False -torch.use_deterministic_algorithms(True) class StableDiffusionImg2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index e46ebeba889f..0a27470b6729 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -14,7 +14,6 @@ # limitations under the License. import gc -import os import random import unittest @@ -38,9 +37,7 @@ from ...test_pipelines_common import PipelineTesterMixin -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" torch.backends.cuda.matmul.allow_tf32 = False -torch.use_deterministic_algorithms(True) class StableDiffusionInpaintPipelineFastTests(PipelineTesterMixin, unittest.TestCase): @@ -168,6 +165,9 @@ def test_stable_diffusion_inpaint_with_num_images_per_prompt(self): @slow @require_torch_gpu class StableDiffusionInpaintPipelineSlowTests(unittest.TestCase): + def setUp(self): + super().setUp() + def tearDown(self): super().tearDown() gc.collect() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py index b4bcd118377c..f11d3848f78e 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py @@ -24,12 +24,13 @@ from diffusers import ( AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, StableDiffusionDepth2ImgPipeline, UNet2DConditionModel, ) -from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device +from diffusers.utils import floats_tensor, load_image, load_numpy, nightly, slow, torch_device from diffusers.utils.import_utils import is_accelerate_available from diffusers.utils.testing_utils import require_torch_gpu from PIL import Image @@ -49,7 +50,7 @@ @unittest.skipIf(torch_device == "mps", reason="The depth model does not support MPS yet") -class StableDiffusiondepth2imgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): +class StableDiffusionDepth2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): pipeline_class = StableDiffusionDepth2ImgPipeline test_save_load_optional_components = False @@ -388,186 +389,217 @@ def test_stable_diffusion_depth2img_pil(self): @slow @require_torch_gpu -class StableDiffusionDepth2ImgPipelineIntegrationTests(unittest.TestCase): +class StableDiffusionDepth2ImgPipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() - def test_stable_diffusion_depth2img_pipeline_default(self): + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.npy" - ) + inputs = { + "prompt": "two tigers", + "image": init_image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs - model_id = "stabilityai/stable-diffusion-2-depth" - pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(model_id) + def test_stable_diffusion_depth2img_pipeline_default(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None + ) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "two tigers" + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - generator=generator, - output_type="np", - ) - image = output.images[0] - - assert image.shape == (480, 640, 3) - # depth2img is flaky across GPUs even in fp32, so using MAE here - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 480, 640, 3) + expected_slice = np.array([0.75446, 0.74692, 0.75951, 0.81611, 0.80593, 0.79992, 0.90529, 0.87921, 0.86903]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 def test_stable_diffusion_depth2img_pipeline_k_lms(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" - ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats_k_lms.npy" + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None ) - - model_id = "stabilityai/stable-diffusion-2-depth" - lms = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(model_id, scheduler=lms) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "two tigers" + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - generator=generator, - output_type="np", - ) - image = output.images[0] - - assert image.shape == (480, 640, 3) - assert np.abs(expected_image - image).max() < 5e-3 + assert image.shape == (1, 480, 640, 3) + expected_slice = np.array([0.63957, 0.64879, 0.65668, 0.64385, 0.67078, 0.63588, 0.66577, 0.62180, 0.66286]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 def test_stable_diffusion_depth2img_pipeline_ddim(self): - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" - ) - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats_ddim.npy" + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None ) - - model_id = "stabilityai/stable-diffusion-2-depth" - ddim = DDIMScheduler.from_pretrained(model_id, subfolder="scheduler") - pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(model_id, scheduler=ddim) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "two tigers" + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - generator=generator, - output_type="np", - ) - image = output.images[0] - - assert image.shape == (480, 640, 3) - assert np.abs(expected_image - image).max() < 1e-3 + assert image.shape == (1, 480, 640, 3) + expected_slice = np.array([0.62840, 0.64191, 0.62953, 0.63653, 0.64205, 0.61574, 0.62252, 0.65827, 0.64809]) + assert np.abs(expected_slice - image_slice).max() < 1e-4 def test_stable_diffusion_depth2img_intermediate_state(self): number_of_steps = 0 - def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: - test_callback_fn.has_been_called = True + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True nonlocal number_of_steps number_of_steps += 1 if step == 1: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 60, 80) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array( - [-0.7825, 0.5786, -0.9125, -0.9885, -1.0071, 2.7126, -0.8490, 0.3776, -0.0791] - ) + expected_slice = np.array([-1.148, -0.2079, -0.622, -2.477, -2.348, 0.3828, -2.055, -1.569, -1.526]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 - elif step == 37: + elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 60, 80) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array( - [-0.6110, -0.2347, -0.5115, -1.1383, -1.4755, -0.5970, -0.9050, -0.7199, -0.8417] - ) - assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-2 + expected_slice = np.array([-1.145, -0.2063, -0.6216, -2.469, -2.344, 0.3794, -2.05, -1.57, -1.521]) + assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 - test_callback_fn.has_been_called = False + callback_fn.has_been_called = False - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None, revision="fp16", torch_dtype=torch.float16 ) - - pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") - pipe.to(torch_device) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "two tigers" - - generator = torch.Generator(device=torch_device).manual_seed(0) - pipe( - prompt=prompt, - image=init_image, - strength=0.75, - num_inference_steps=50, - generator=generator, - callback=test_callback_fn, - callback_steps=1, - ) - assert test_callback_fn.has_been_called - assert number_of_steps == 37 + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 2 def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() torch.cuda.reset_peak_memory_stats() - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" - ) - init_image = init_image.resize((768, 512)) - - model_id = "stabilityai/stable-diffusion-2-depth" - lms = LMSDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( - model_id, scheduler=lms, safety_checker=None, revision="fp16", torch_dtype=torch.float16 + "stabilityai/stable-diffusion-2-depth", safety_checker=None, revision="fp16", torch_dtype=torch.float16 ) - pipe.to(torch_device) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing(1) pipe.enable_sequential_cpu_offload() - prompt = "A fantasy landscape, trending on artstation" - - generator = torch.Generator(device=torch_device).manual_seed(0) - _ = pipe( - prompt=prompt, - image=init_image, - strength=0.75, - guidance_scale=7.5, - generator=generator, - output_type="np", - num_inference_steps=2, - ) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.9 GB is allocated assert mem_bytes < 2.9 * 10**9 + + +@nightly +@require_torch_gpu +class StableDiffusionImg2ImgPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" + ) + inputs = { + "prompt": "two tigers", + "image": init_image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_depth2img_pndm(self): + sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_pndm.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_depth2img_ddim(self): + sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_ddim.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_img2img_lms(self): + sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_lms.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) + + def test_img2img_dpm(self): + sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 30 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_dpm_multi.npy" + ) + + assert np.allclose(image, expected_image, atol=1e-3) From 1286a4061e5093df437a33e7814d3bb63caa93b1 Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 09:38:23 +0100 Subject: [PATCH 14/22] allclose -> max diff for shorter logs --- .../stable_diffusion/test_stable_diffusion.py | 24 +++++++++---------- .../test_stable_diffusion_image_variation.py | 8 +++---- .../test_stable_diffusion_img2img.py | 16 ++++++------- .../test_stable_diffusion_inpaint.py | 16 ++++++------- .../test_stable_diffusion_inpaint_legacy.py | 16 ++++++------- .../test_stable_diffusion_depth.py | 16 ++++++------- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index a5e7f8ecf3f4..a8c0828881da 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -713,8 +713,8 @@ def test_stable_diffusion_1_4_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_text2img/stable_diffusion_1_4_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_stable_diffusion_1_5_pndm(self): sd_pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5").to(torch_device) @@ -727,8 +727,8 @@ def test_stable_diffusion_1_5_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_text2img/stable_diffusion_1_5_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_stable_diffusion_ddim(self): sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) @@ -742,8 +742,8 @@ def test_stable_diffusion_ddim(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_text2img/stable_diffusion_1_4_ddim.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_stable_diffusion_lms(self): sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) @@ -757,8 +757,8 @@ def test_stable_diffusion_lms(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_text2img/stable_diffusion_1_4_lms.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_stable_diffusion_euler(self): sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) @@ -772,8 +772,8 @@ def test_stable_diffusion_euler(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_text2img/stable_diffusion_1_4_euler.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_stable_diffusion_dpm(self): sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) @@ -788,5 +788,5 @@ def test_stable_diffusion_dpm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_text2img/stable_diffusion_1_4_dpm_multi.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index 7824689da0dd..eef1cb7666ac 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -311,8 +311,8 @@ def test_img_variation_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_imgvar/lambdalabs_variations_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_img_variation_dpm(self): sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("fusing/sd-image-variations-diffusers") @@ -328,5 +328,5 @@ def test_img_variation_dpm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_imgvar/lambdalabs_variations_dpm_multi.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index cbc94451ae27..3a7387d80b60 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -374,8 +374,8 @@ def test_img2img_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_img2img/stable_diffusion_1_5_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_img2img_ddim(self): sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") @@ -390,8 +390,8 @@ def test_img2img_ddim(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_img2img/stable_diffusion_1_5_ddim.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_img2img_lms(self): sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") @@ -406,8 +406,8 @@ def test_img2img_lms(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_img2img/stable_diffusion_1_5_lms.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_img2img_dpm(self): sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") @@ -423,5 +423,5 @@ def test_img2img_dpm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_img2img/stable_diffusion_1_5_dpm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index 0a27470b6729..340b7b71f0da 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -322,8 +322,8 @@ def test_inpaint_ddim(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint/stable_diffusion_inpaint_ddim.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_inpaint_pndm(self): sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") @@ -338,8 +338,8 @@ def test_inpaint_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint/stable_diffusion_inpaint_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_inpaint_lms(self): sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") @@ -354,8 +354,8 @@ def test_inpaint_lms(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint/stable_diffusion_inpaint_lms.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_inpaint_dpm(self): sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") @@ -371,8 +371,8 @@ def test_inpaint_dpm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint/stable_diffusion_inpaint_dpm_multi.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 class StableDiffusionInpaintingPrepareMaskAndMaskedImageTests(unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py index da589f9ba94e..873252054bfa 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py @@ -479,8 +479,8 @@ def test_inpaint_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_inpaint_ddim(self): sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") @@ -495,8 +495,8 @@ def test_inpaint_ddim(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_ddim.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_inpaint_lms(self): sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") @@ -511,8 +511,8 @@ def test_inpaint_lms(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_lms.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_inpaint_dpm(self): sd_pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained("runwayml/stable-diffusion-v1-5") @@ -528,5 +528,5 @@ def test_inpaint_dpm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_inpaint_legacy/stable_diffusion_1_5_dpm_multi.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py index f11d3848f78e..cc3ecbeab0eb 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py @@ -552,8 +552,8 @@ def test_depth2img_pndm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_depth2img/stable_diffusion_2_0_pndm.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_depth2img_ddim(self): sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") @@ -568,8 +568,8 @@ def test_depth2img_ddim(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_depth2img/stable_diffusion_2_0_ddim.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_img2img_lms(self): sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") @@ -584,8 +584,8 @@ def test_img2img_lms(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_depth2img/stable_diffusion_2_0_lms.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 def test_img2img_dpm(self): sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") @@ -601,5 +601,5 @@ def test_img2img_dpm(self): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_depth2img/stable_diffusion_2_0_dpm_multi.npy" ) - - assert np.allclose(image, expected_image, atol=1e-3) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 From ba7c1bf2167945024269d97d40edcb1a46aec4c8 Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 10:21:52 +0100 Subject: [PATCH 15/22] add SD2 --- .github/workflows/nightly_tests.yml | 26 +- .github/workflows/push_tests.yml | 10 + .../stable_diffusion/test_stable_diffusion.py | 6 +- .../test_stable_diffusion.py | 335 ++++++++++-------- .../test_stable_diffusion_depth.py | 84 ++--- 5 files changed, 265 insertions(+), 196 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index a4f65f916bd5..05a1cec73811 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -17,16 +17,26 @@ env: RUN_NIGHTLY: yes jobs: - run_slow_tests: + run_nightly_tests: strategy: fail-fast: false matrix: config: - - name: Slow PyTorch CUDA tests on Ubuntu + - name: Nightly PyTorch CUDA tests on Ubuntu framework: pytorch runner: docker-gpu image: diffusers/diffusers-pytorch-cuda report: torch_cuda + - name: Nightly Flax TPU tests on Ubuntu + framework: flax + runner: docker-tpu + image: diffusers/diffusers-flax-tpu + report: flax_tpu + - name: Nightly ONNXRuntime CUDA tests on Ubuntu + framework: onnxruntime + runner: docker-gpu + image: diffusers/diffusers-onnxruntime-cuda + report: onnx_cuda name: ${{ matrix.config.name }} @@ -61,7 +71,7 @@ jobs: run: | python utils/print_env.py - - name: Run slow PyTorch CUDA tests + - name: Run nightly PyTorch CUDA tests if: ${{ matrix.config.framework == 'pytorch' }} env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} @@ -71,7 +81,7 @@ jobs: --make-reports=tests_${{ matrix.config.report }} \ tests/ - - name: Run slow Flax TPU tests + - name: Run nightly Flax TPU tests if: ${{ matrix.config.framework == 'flax' }} env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} @@ -81,7 +91,7 @@ jobs: --make-reports=tests_${{ matrix.config.report }} \ tests/ - - name: Run slow ONNXRuntime CUDA tests + - name: Run nightly ONNXRuntime CUDA tests if: ${{ matrix.config.framework == 'onnxruntime' }} env: HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} @@ -102,8 +112,8 @@ jobs: name: ${{ matrix.config.report }}_test_reports path: reports - run_slow_tests_apple_m1: - name: Slow PyTorch MPS tests on MacOS + run_nightly_tests_apple_m1: + name: Nightly PyTorch MPS tests on MacOS runs-on: [ self-hosted, apple-m1 ] steps: @@ -135,7 +145,7 @@ jobs: run: | ${CONDA_RUN} python utils/print_env.py - - name: Run slow PyTorch tests on M1 (MPS) + - name: Run nightly PyTorch tests on M1 (MPS) shell: arch -arch arm64 bash {0} env: HF_HOME: /System/Volumes/Data/mnt/cache diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 288ed1fe3f9b..283b725169cd 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -27,6 +27,16 @@ jobs: runner: docker-gpu image: diffusers/diffusers-pytorch-cuda report: torch_cuda + - name: Slow Flax TPU tests on Ubuntu + framework: flax + runner: docker-tpu + image: diffusers/diffusers-flax-tpu + report: flax_tpu + - name: Slow ONNXRuntime CUDA tests on Ubuntu + framework: onnxruntime + runner: docker-gpu + image: diffusers/diffusers-onnxruntime-cuda + report: onnx_cuda name: ${{ matrix.config.name }} diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index a8c0828881da..0df2484df15d 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -44,7 +44,7 @@ torch.backends.cuda.matmul.allow_tf32 = False -class StableDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): +class StableDiffusion2PipelineFastTests(PipelineTesterMixin, unittest.TestCase): pipeline_class = StableDiffusionPipeline def get_dummy_components(self): @@ -437,7 +437,7 @@ def test_stable_diffusion_height_width_opt(self): @slow @require_torch_gpu -class StableDiffusionPipelineSlowTests(unittest.TestCase): +class StableDiffusion2PipelineSlowTests(unittest.TestCase): def tearDown(self): super().tearDown() gc.collect() @@ -682,7 +682,7 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): @nightly @require_torch_gpu -class StableDiffusionPipelineNightlyTests(unittest.TestCase): +class StableDiffusion2PipelineNightlyTests(unittest.TestCase): def tearDown(self): super().tearDown() gc.collect() diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py index 5f8644754204..d486ea092620 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py @@ -22,6 +22,7 @@ from diffusers import ( AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler, @@ -30,7 +31,7 @@ UNet2DConditionModel, logging, ) -from diffusers.utils import load_numpy, slow, torch_device +from diffusers.utils import load_numpy, nightly, slow, torch_device from diffusers.utils.testing_utils import CaptureLogger, require_torch_gpu from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer @@ -239,170 +240,116 @@ def test_stable_diffusion_long_prompt(self): @slow @require_torch_gpu -class StableDiffusion2PipelineIntegrationTests(unittest.TestCase): +class StableDiffusion2PipelineSlowTests(unittest.TestCase): def tearDown(self): - # clean up the VRAM after each test super().tearDown() gc.collect() torch.cuda.empty_cache() - def test_stable_diffusion(self): - sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") - sd_pipe = sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs - prompt = "A painting of a squirrel eating a burger" - generator = torch.Generator(device=torch_device).manual_seed(0) - output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=20, output_type="np") + def test_stable_diffusion_default_ddim(self): + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) - image = output.images - image_slice = image[0, 253:256, 253:256, -1] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.0788, 0.0823, 0.1091, 0.1165, 0.1263, 0.1459, 0.1317, 0.1507, 0.1551]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 - - def test_stable_diffusion_ddim(self): - scheduler = DDIMScheduler.from_pretrained("stabilityai/stable-diffusion-2-base", subfolder="scheduler") - sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base", scheduler=scheduler) - sd_pipe = sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + expected_slice = np.array([0.49493, 0.47896, 0.40798, 0.54214, 0.53212, 0.48202, 0.47656, 0.46329, 0.48506]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 - prompt = "A painting of a squirrel eating a burger" - generator = torch.Generator(device=torch_device).manual_seed(0) - - output = sd_pipe([prompt], generator=generator, num_inference_steps=5, output_type="numpy") - image = output.images + def test_stable_diffusion_pndm(self): + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) - image_slice = image[0, 253:256, 253:256, -1] + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.0642, 0.0382, 0.0408, 0.0395, 0.0227, 0.0942, 0.0749, 0.0669, 0.0248]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + expected_slice = np.array([0.49493, 0.47896, 0.40798, 0.54214, 0.53212, 0.48202, 0.47656, 0.46329, 0.48506]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 def test_stable_diffusion_k_lms(self): - scheduler = LMSDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-2-base", subfolder="scheduler") - sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base", scheduler=scheduler) - sd_pipe = sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) - prompt = "a photograph of an astronaut riding a horse" - generator = torch.Generator(device=torch_device).manual_seed(0) - image = sd_pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=5, output_type="numpy" - ).images + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() - image_slice = image[0, 253:256, 253:256, -1] assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.0548, 0.0626, 0.0612, 0.0611, 0.0706, 0.0586, 0.0843, 0.0333, 0.1197]) - assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + expected_slice = np.array([0.10440, 0.13115, 0.11100, 0.10141, 0.11440, 0.07215, 0.11332, 0.09693, 0.10006]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 def test_stable_diffusion_attention_slicing(self): torch.cuda.reset_peak_memory_stats() - model_id = "stabilityai/stable-diffusion-2-base" - pipe = StableDiffusionPipeline.from_pretrained(model_id, revision="fp16", torch_dtype=torch.float16) - pipe.to(torch_device) + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", revision="fp16", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) - prompt = "a photograph of an astronaut riding a horse" - - # make attention efficient + # enable attention slicing pipe.enable_attention_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - output_chunked = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image_chunked = output_chunked.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image_sliced = pipe(**inputs).images mem_bytes = torch.cuda.max_memory_allocated() torch.cuda.reset_peak_memory_stats() - # make sure that less than 3.75 GB is allocated - assert mem_bytes < 3.75 * 10**9 + # make sure that less than 3.3 GB is allocated + assert mem_bytes < 3.3 * 10**9 - # disable chunking + # disable slicing pipe.disable_attention_slicing() - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - output = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image = output.images + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images - # make sure that more than 3.75 GB is allocated + # make sure that more than 3.3 GB is allocated mem_bytes = torch.cuda.max_memory_allocated() - assert mem_bytes > 3.75 * 10**9 - assert np.abs(image_chunked.flatten() - image.flatten()).max() < 1e-3 - - def test_stable_diffusion_same_quality(self): - torch.cuda.reset_peak_memory_stats() - model_id = "stabilityai/stable-diffusion-2-base" - pipe = StableDiffusionPipeline.from_pretrained(model_id, revision="fp16", torch_dtype=torch.float16) - pipe = pipe.to(torch_device) - pipe.enable_attention_slicing() - pipe.set_progress_bar_config(disable=None) - - prompt = "a photograph of an astronaut riding a horse" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output_chunked = pipe( - [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" - ) - image_chunked = output_chunked.images - - pipe = StableDiffusionPipeline.from_pretrained(model_id) - pipe = pipe.to(torch_device) - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe([prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy") - image = output.images - - # Make sure results are close enough - diff = np.abs(image_chunked.flatten() - image.flatten()) - # They ARE different since ops are not run always at the same precision - # however, they should be extremely close. - assert diff.mean() < 5e-2 - - def test_stable_diffusion_text2img_pipeline_default(self): - expected_image = load_numpy( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-text2img/astronaut_riding_a_horse.npy" - ) - - model_id = "stabilityai/stable-diffusion-2-base" - pipe = StableDiffusionPipeline.from_pretrained(model_id, safety_checker=None) - pipe.to(torch_device) - pipe.set_progress_bar_config(disable=None) - pipe.enable_attention_slicing() - - prompt = "astronaut riding a horse" - - generator = torch.Generator(device=torch_device).manual_seed(0) - output = pipe(prompt=prompt, guidance_scale=7.5, generator=generator, output_type="np") - image = output.images[0] - - assert image.shape == (512, 512, 3) - assert np.abs(expected_image - image).max() < 5e-3 + assert mem_bytes > 3.3 * 10**9 + assert np.abs(image_sliced - image).max() < 1e-3 def test_stable_diffusion_text2img_intermediate_state(self): number_of_steps = 0 - def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: - test_callback_fn.has_been_called = True + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True nonlocal number_of_steps number_of_steps += 1 - if step == 0: + if step == 1: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([1.8606, 1.3169, -0.0691, 1.2374, -2.309, 1.077, -0.1084, -0.6774, -2.9594]) + expected_slice = np.array([-0.3857, -0.4507, -1.167, 0.074, -1.108, 0.7183, -1.822, 0.1915, 1.283]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-3 - elif step == 20: + elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([1.0757, 1.1860, 1.1410, 0.4645, -0.2476, 0.6100, -0.7755, -0.8841, -0.9497]) - assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + expected_slice = np.array([0.268, -0.2095, -0.7744, -0.541, -0.79, 0.3926, -0.7754, 0.465, 1.291]) + assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-2 - test_callback_fn.has_been_called = False + callback_fn.has_been_called = False pipe = StableDiffusionPipeline.from_pretrained( "stabilityai/stable-diffusion-2-base", revision="fp16", torch_dtype=torch.float16 @@ -411,37 +358,139 @@ def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> No pipe.set_progress_bar_config(disable=None) pipe.enable_attention_slicing() - prompt = "Andromeda galaxy in a bottle" - - generator = torch.Generator(device=torch_device).manual_seed(0) - with torch.autocast(torch_device): - pipe( - prompt=prompt, - num_inference_steps=20, - guidance_scale=7.5, - generator=generator, - callback=test_callback_fn, - callback_steps=1, - ) - assert test_callback_fn.has_been_called - assert number_of_steps == 20 + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == inputs["num_inference_steps"] def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() torch.cuda.reset_peak_memory_stats() - pipeline_id = "stabilityai/stable-diffusion-2-base" - prompt = "Andromeda galaxy in a bottle" - - pipeline = StableDiffusionPipeline.from_pretrained(pipeline_id, revision="fp16", torch_dtype=torch.float16) - pipeline = pipeline.to(torch_device) - pipeline.enable_attention_slicing(1) - pipeline.enable_sequential_cpu_offload() + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", revision="fp16", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() - generator = torch.Generator(device=torch_device).manual_seed(0) - _ = pipeline(prompt, generator=generator, num_inference_steps=5) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) mem_bytes = torch.cuda.max_memory_allocated() # make sure that less than 2.8 GB is allocated assert mem_bytes < 2.8 * 10**9 + + +@nightly +@require_torch_gpu +class StableDiffusionPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_2_0_default_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_0_base_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_2_1_default_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_lms(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_lms.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_euler(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = EulerDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_euler.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_dpm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_dpm_multi.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py index cc3ecbeab0eb..d37bd3f944d1 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py @@ -276,12 +276,12 @@ def test_progress_bar(self): def test_stable_diffusion_depth2img_default_case(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator components = self.get_dummy_components() - sd_pipe = StableDiffusionDepth2ImgPipeline(**components) - sd_pipe = sd_pipe.to(device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_dummy_inputs(device) - image = sd_pipe(**inputs).images + image = pipe(**inputs).images image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) @@ -294,13 +294,13 @@ def test_stable_diffusion_depth2img_default_case(self): def test_stable_diffusion_depth2img_negative_prompt(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator components = self.get_dummy_components() - sd_pipe = StableDiffusionDepth2ImgPipeline(**components) - sd_pipe = sd_pipe.to(device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_dummy_inputs(device) negative_prompt = "french fries" - output = sd_pipe(**inputs, negative_prompt=negative_prompt) + output = pipe(**inputs, negative_prompt=negative_prompt) image = output.images image_slice = image[0, -3:, -3:, -1] @@ -314,14 +314,14 @@ def test_stable_diffusion_depth2img_negative_prompt(self): def test_stable_diffusion_depth2img_multiple_init_images(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator components = self.get_dummy_components() - sd_pipe = StableDiffusionDepth2ImgPipeline(**components) - sd_pipe = sd_pipe.to(device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_dummy_inputs(device) inputs["prompt"] = [inputs["prompt"]] * 2 inputs["image"] = 2 * [inputs["image"]] - image = sd_pipe(**inputs).images + image = pipe(**inputs).images image_slice = image[-1, -3:, -3:, -1] assert image.shape == (2, 32, 32, 3) @@ -335,13 +335,13 @@ def test_stable_diffusion_depth2img_multiple_init_images(self): def test_stable_diffusion_depth2img_num_images_per_prompt(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator components = self.get_dummy_components() - sd_pipe = StableDiffusionDepth2ImgPipeline(**components) - sd_pipe = sd_pipe.to(device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) # test num_images_per_prompt=1 (default) inputs = self.get_dummy_inputs(device) - images = sd_pipe(**inputs).images + images = pipe(**inputs).images assert images.shape == (1, 32, 32, 3) @@ -349,14 +349,14 @@ def test_stable_diffusion_depth2img_num_images_per_prompt(self): batch_size = 2 inputs = self.get_dummy_inputs(device) inputs["prompt"] = [inputs["prompt"]] * batch_size - images = sd_pipe(**inputs).images + images = pipe(**inputs).images assert images.shape == (batch_size, 32, 32, 3) # test num_images_per_prompt for single prompt num_images_per_prompt = 2 inputs = self.get_dummy_inputs(device) - images = sd_pipe(**inputs, num_images_per_prompt=num_images_per_prompt).images + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt).images assert images.shape == (num_images_per_prompt, 32, 32, 3) @@ -364,20 +364,20 @@ def test_stable_diffusion_depth2img_num_images_per_prompt(self): batch_size = 2 inputs = self.get_dummy_inputs(device) inputs["prompt"] = [inputs["prompt"]] * batch_size - images = sd_pipe(**inputs, num_images_per_prompt=num_images_per_prompt).images + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt).images assert images.shape == (batch_size * num_images_per_prompt, 32, 32, 3) def test_stable_diffusion_depth2img_pil(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator components = self.get_dummy_components() - sd_pipe = StableDiffusionDepth2ImgPipeline(**components) - sd_pipe = sd_pipe.to(device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_dummy_inputs(device) - image = sd_pipe(**inputs).images + image = pipe(**inputs).images image_slice = image[0, -3:, -3:, -1] if torch_device == "mps": @@ -541,12 +541,12 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): return inputs def test_depth2img_pndm(self): - sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") - sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_inputs(torch_device) - image = sd_pipe(**inputs).images[0] + image = pipe(**inputs).images[0] expected_image = load_numpy( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" @@ -556,13 +556,13 @@ def test_depth2img_pndm(self): assert max_diff < 1e-3 def test_depth2img_ddim(self): - sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") - sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) - sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_inputs(torch_device) - image = sd_pipe(**inputs).images[0] + image = pipe(**inputs).images[0] expected_image = load_numpy( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" @@ -572,13 +572,13 @@ def test_depth2img_ddim(self): assert max_diff < 1e-3 def test_img2img_lms(self): - sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") - sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) - sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_inputs(torch_device) - image = sd_pipe(**inputs).images[0] + image = pipe(**inputs).images[0] expected_image = load_numpy( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" @@ -588,14 +588,14 @@ def test_img2img_lms(self): assert max_diff < 1e-3 def test_img2img_dpm(self): - sd_pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") - sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) - sd_pipe.to(torch_device) - sd_pipe.set_progress_bar_config(disable=None) + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) inputs = self.get_inputs(torch_device) inputs["num_inference_steps"] = 30 - image = sd_pipe(**inputs).images[0] + image = pipe(**inputs).images[0] expected_image = load_numpy( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" From 0f64d886f81b30338c47b5c634501d6a71690096 Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 10:33:52 +0100 Subject: [PATCH 16/22] debug --- .github/workflows/nightly_tests.yml | 6 +++--- .github/workflows/push_tests.yml | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index 05a1cec73811..2b8b5796d155 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -3,9 +3,9 @@ name: Nightly tests on main on: schedule: - cron: "0 0 * * *" # every day at midnight -# pull_request: -# branches: -# - main + pull_request: # TODO: only for debugging, remove before merging! + branches: + - main env: DIFFUSERS_IS_CI: yes diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 283b725169cd..886c6283336c 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -4,9 +4,6 @@ on: push: branches: - main -# pull_request: -# branches: -# - main env: DIFFUSERS_IS_CI: yes From 89b2d5fcab5f28930fbe29df68cb65c89266fd5e Mon Sep 17 00:00:00 2001 From: Anton Lozhkov Date: Fri, 16 Dec 2022 10:59:55 +0100 Subject: [PATCH 17/22] Update tests/pipelines/stable_diffusion_2/test_stable_diffusion.py Co-authored-by: Patrick von Platen --- tests/pipelines/stable_diffusion_2/test_stable_diffusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py index d486ea092620..328e0a2d3cf4 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py @@ -386,7 +386,7 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): @nightly @require_torch_gpu -class StableDiffusionPipelineNightlyTests(unittest.TestCase): +class StableDiffusion2PipelineNightlyTests(unittest.TestCase): def tearDown(self): super().tearDown() gc.collect() From 3df885e3acd0d9cab91124f5a851280f688e7b9c Mon Sep 17 00:00:00 2001 From: Anton Lozhkov Date: Fri, 16 Dec 2022 11:19:24 +0100 Subject: [PATCH 18/22] Update tests/pipelines/stable_diffusion/test_stable_diffusion.py Co-authored-by: Patrick von Platen --- tests/pipelines/stable_diffusion/test_stable_diffusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 0df2484df15d..a5852ade3408 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -682,7 +682,7 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): @nightly @require_torch_gpu -class StableDiffusion2PipelineNightlyTests(unittest.TestCase): +class StableDiffusionPipelineNightlyTests(unittest.TestCase): def tearDown(self): super().tearDown() gc.collect() From bed3dac721e418460103edb06279b1afffa139a1 Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 17:06:07 +0100 Subject: [PATCH 19/22] fix refs --- .../stable_diffusion/test_onnx_stable_diffusion.py | 4 ++-- .../test_onnx_stable_diffusion_img2img.py | 10 ++++++++-- .../test_onnx_stable_diffusion_inpaint.py | 10 ++++++++-- .../test_onnx_stable_diffusion_inpaint_legacy.py | 4 ++-- .../test_stable_diffusion_image_variation.py | 11 +++++++---- .../stable_diffusion/test_stable_diffusion_img2img.py | 10 +++++----- 6 files changed, 32 insertions(+), 17 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py index ba60b0477e12..f076d726060f 100644 --- a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py @@ -27,7 +27,7 @@ OnnxStableDiffusionPipeline, PNDMScheduler, ) -from diffusers.utils.testing_utils import is_onnx_available, require_onnxruntime, require_torch_gpu, slow +from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu from ...test_pipelines_onnx_common import OnnxPipelineTesterMixin @@ -128,7 +128,7 @@ def test_pipeline_dpm_multistep(self): assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 -@slow +@nightly @require_onnxruntime @require_torch_gpu class OnnxStableDiffusionPipelineIntegrationTests(unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py index 10dd0876c989..ad33dda680fe 100644 --- a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py @@ -27,7 +27,13 @@ PNDMScheduler, ) from diffusers.utils import floats_tensor -from diffusers.utils.testing_utils import is_onnx_available, load_image, require_onnxruntime, require_torch_gpu, slow +from diffusers.utils.testing_utils import ( + is_onnx_available, + load_image, + nightly, + require_onnxruntime, + require_torch_gpu, +) from ...test_pipelines_onnx_common import OnnxPipelineTesterMixin @@ -134,7 +140,7 @@ def test_pipeline_dpm_multistep(self): assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 -@slow +@nightly @require_onnxruntime @require_torch_gpu class OnnxStableDiffusionImg2ImgPipelineIntegrationTests(unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py index ae87cd8a869c..cec4394b3452 100644 --- a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py @@ -18,7 +18,13 @@ import numpy as np from diffusers import LMSDiscreteScheduler, OnnxStableDiffusionInpaintPipeline -from diffusers.utils.testing_utils import is_onnx_available, load_image, require_onnxruntime, require_torch_gpu, slow +from diffusers.utils.testing_utils import ( + is_onnx_available, + load_image, + nightly, + require_onnxruntime, + require_torch_gpu, +) from ...test_pipelines_onnx_common import OnnxPipelineTesterMixin @@ -32,7 +38,7 @@ class OnnxStableDiffusionPipelineFastTests(OnnxPipelineTesterMixin, unittest.Tes pass -@slow +@nightly @require_onnxruntime @require_torch_gpu class OnnxStableDiffusionInpaintPipelineIntegrationTests(unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint_legacy.py b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint_legacy.py index d99ab074e869..0237970ad2f3 100644 --- a/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint_legacy.py +++ b/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint_legacy.py @@ -22,9 +22,9 @@ is_onnx_available, load_image, load_numpy, + nightly, require_onnxruntime, require_torch_gpu, - slow, ) @@ -32,7 +32,7 @@ import onnxruntime as ort -@slow +@nightly @require_onnxruntime @require_torch_gpu class StableDiffusionOnnxInpaintLegacyPipelineIntegrationTests(unittest.TestCase): diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index eef1cb7666ac..01ad9236f80d 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -203,7 +203,9 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): return inputs def test_stable_diffusion_img_variation_pipeline_default(self): - sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("lambdalabs/sd-image-variations-diffusers") + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", safety_checker=None + ) sd_pipe = sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) @@ -212,7 +214,7 @@ def test_stable_diffusion_img_variation_pipeline_default(self): image_slice = image[0, -3:, -3:, -1].flatten() assert image.shape == (1, 512, 512, 3) - expected_slice = np.array([0.43625, 0.43554, 0.36670, 0.40660, 0.39703, 0.38658, 0.43936, 0.43557, 0.40592]) + expected_slice = np.array([0.84491, 0.90789, 0.75708, 0.78734, 0.83485, 0.70099, 0.66938, 0.68727, 0.61379]) assert np.abs(image_slice - expected_slice).max() < 1e-4 def test_stable_diffusion_img_variation_intermediate_state(self): @@ -226,19 +228,20 @@ def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([1.83, 1.293, -0.09705, 1.256, -2.293, 1.091, -0.0809, -0.65, -2.953]) + expected_slice = np.array([-0.1572, 0.2837, -0.798, -0.1201, -1.304, 0.7754, -2.12, 0.0443, 1.627]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-3 elif step == 2: latents = latents.detach().cpu().numpy() assert latents.shape == (1, 4, 64, 64) latents_slice = latents[0, -3:, -3:, -1] - expected_slice = np.array([2.285, 2.703, 1.969, 0.696, -1.323, 0.9253, -0.5464, -1.521, -2.537]) + expected_slice = np.array([0.6143, 1.734, 1.158, -2.145, -1.926, 0.748, -0.7246, 0.994, 1.539]) assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 callback_fn.has_been_called = False pipe = StableDiffusionImageVariationPipeline.from_pretrained( "fusing/sd-image-variations-diffusers", + safety_checker=None, torch_dtype=torch.float16, ) pipe.to(torch_device) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index 3a7387d80b60..3e02b82e9f11 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -223,7 +223,7 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" - "/stable_diffusion_img2img/sketch-mountains-input.jpg" + "/stable_diffusion_img2img/sketch-mountains-input.png" ) init_image = init_image.resize((768, 512)) inputs = { @@ -249,7 +249,7 @@ def test_stable_diffusion_img2img_default(self): assert image.shape == (1, 512, 768, 3) expected_slice = np.array([0.27150, 0.14849, 0.15605, 0.26740, 0.16954, 0.18204, 0.31470, 0.26311, 0.24525]) - assert np.abs(expected_slice - image_slice).max() < 1e-4 + assert np.abs(expected_slice - image_slice).max() < 1e-3 def test_stable_diffusion_img2img_k_lms(self): pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) @@ -264,7 +264,7 @@ def test_stable_diffusion_img2img_k_lms(self): assert image.shape == (1, 512, 768, 3) expected_slice = np.array([0.04890, 0.04862, 0.06422, 0.04655, 0.05108, 0.05307, 0.05926, 0.08759, 0.06852]) - assert np.abs(expected_slice - image_slice).max() < 1e-4 + assert np.abs(expected_slice - image_slice).max() < 1e-3 def test_stable_diffusion_img2img_ddim(self): pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) @@ -279,7 +279,7 @@ def test_stable_diffusion_img2img_ddim(self): assert image.shape == (1, 512, 768, 3) expected_slice = np.array([0.06069, 0.05703, 0.08054, 0.05797, 0.06286, 0.06234, 0.08438, 0.11151, 0.08068]) - assert np.abs(expected_slice - image_slice).max() < 1e-4 + assert np.abs(expected_slice - image_slice).max() < 1e-3 def test_stable_diffusion_img2img_intermediate_state(self): number_of_steps = 0 @@ -348,7 +348,7 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" - "/stable_diffusion_img2img/sketch-mountains-input.jpg" + "/stable_diffusion_img2img/sketch-mountains-input.png" ) init_image = init_image.resize((768, 512)) inputs = { From 63105dbf8593db0559c55deafa8d91584dd34c66 Mon Sep 17 00:00:00 2001 From: Anton Lozhkov Date: Fri, 16 Dec 2022 17:06:54 +0100 Subject: [PATCH 20/22] Update src/diffusers/utils/testing_utils.py Co-authored-by: Pedro Cuenca --- src/diffusers/utils/testing_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/utils/testing_utils.py b/src/diffusers/utils/testing_utils.py index 3622002889b1..a604aed605ce 100644 --- a/src/diffusers/utils/testing_utils.py +++ b/src/diffusers/utils/testing_utils.py @@ -116,7 +116,7 @@ def nightly(test_case): """ Decorator marking a test that runs nightly in the diffusers CI. - Slow tests are skipped by default. Set the RUN_SLOW environment variable to a truthy value to run them. + Slow tests are skipped by default. Set the RUN_NIGHTLY environment variable to a truthy value to run them. """ return unittest.skipUnless(_run_nightly_tests, "test is nightly")(test_case) From 130c5c55c61f680515af6b04ac84390fb4d04672 Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 17:45:48 +0100 Subject: [PATCH 21/22] fix refs --- tests/pipelines/repaint/test_repaint.py | 2 ++ tests/pipelines/stable_diffusion/test_stable_diffusion.py | 4 ++-- .../test_stable_diffusion_image_variation.py | 6 ++---- .../stable_diffusion/test_stable_diffusion_img2img.py | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/pipelines/repaint/test_repaint.py b/tests/pipelines/repaint/test_repaint.py index 55148eb54a1b..2288fa63bf5e 100644 --- a/tests/pipelines/repaint/test_repaint.py +++ b/tests/pipelines/repaint/test_repaint.py @@ -110,6 +110,8 @@ def test_celebahq(self): scheduler = RePaintScheduler.from_pretrained(model_id) repaint = RePaintPipeline(unet=unet, scheduler=scheduler).to(torch_device) + repaint.set_progress_bar_config(disable=None) + repaint.enable_attention_slicing() generator = torch.Generator(device=torch_device).manual_seed(0) output = repaint( diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index a5852ade3408..a8c0828881da 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -44,7 +44,7 @@ torch.backends.cuda.matmul.allow_tf32 = False -class StableDiffusion2PipelineFastTests(PipelineTesterMixin, unittest.TestCase): +class StableDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): pipeline_class = StableDiffusionPipeline def get_dummy_components(self): @@ -437,7 +437,7 @@ def test_stable_diffusion_height_width_opt(self): @slow @require_torch_gpu -class StableDiffusion2PipelineSlowTests(unittest.TestCase): +class StableDiffusionPipelineSlowTests(unittest.TestCase): def tearDown(self): super().tearDown() gc.collect() diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py index 01ad9236f80d..bfe56dfb89bb 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_image_variation.py @@ -187,9 +187,8 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" - "/stable_diffusion_imgvar/input_image.jpg" + "/stable_diffusion_imgvar/input_image_vermeer.png" ) - init_image = init_image.resize((512, 512)) latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) latents = torch.from_numpy(latents).to(device=device, dtype=dtype) inputs = { @@ -287,9 +286,8 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): generator = torch.Generator(device=device).manual_seed(seed) init_image = load_image( "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" - "/stable_diffusion_imgvar/input_image.jpg" + "/stable_diffusion_imgvar/input_image_vermeer.png" ) - init_image = init_image.resize((512, 512)) latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) latents = torch.from_numpy(latents).to(device=device, dtype=dtype) inputs = { diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index 3e02b82e9f11..f2949cb3ef19 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -225,7 +225,6 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_img2img/sketch-mountains-input.png" ) - init_image = init_image.resize((768, 512)) inputs = { "prompt": "a fantasy landscape, concept art, high resolution", "image": init_image, @@ -350,7 +349,6 @@ def get_inputs(self, device, dtype=torch.float32, seed=0): "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" "/stable_diffusion_img2img/sketch-mountains-input.png" ) - init_image = init_image.resize((768, 512)) inputs = { "prompt": "a fantasy landscape, concept art, high resolution", "image": init_image, From f7b1fd5c47621b3067d7f9c59817cd1550f0852e Mon Sep 17 00:00:00 2001 From: anton- Date: Fri, 16 Dec 2022 18:43:21 +0100 Subject: [PATCH 22/22] remove debug --- .github/workflows/nightly_tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index 2b8b5796d155..f0185855523e 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -3,9 +3,6 @@ name: Nightly tests on main on: schedule: - cron: "0 0 * * *" # every day at midnight - pull_request: # TODO: only for debugging, remove before merging! - branches: - - main env: DIFFUSERS_IS_CI: yes