From 80af99a34d2474388b1088d3c67b6bf8f4d1e048 Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 18:04:38 +0100 Subject: [PATCH 1/9] Match the generator device to the pipeline for DDPM and DDIM --- src/diffusers/pipelines/ddim/pipeline_ddim.py | 29 +++++++----- src/diffusers/pipelines/ddpm/pipeline_ddpm.py | 15 ++++++- src/diffusers/schedulers/scheduling_ddpm.py | 14 ++++-- tests/pipelines/ddim/test_ddim.py | 11 +++-- tests/pipelines/ddpm/test_ddpm.py | 12 ++--- tests/test_pipelines.py | 44 +++---------------- 6 files changed, 62 insertions(+), 63 deletions(-) diff --git a/src/diffusers/pipelines/ddim/pipeline_ddim.py b/src/diffusers/pipelines/ddim/pipeline_ddim.py index aab6e68613c4..2610f9d3f52a 100644 --- a/src/diffusers/pipelines/ddim/pipeline_ddim.py +++ b/src/diffusers/pipelines/ddim/pipeline_ddim.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect from typing import Optional, Tuple, Union import torch from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from ...utils import deprecate class DDIMPipeline(DiffusionPipeline): @@ -75,24 +75,29 @@ def __call__( generated images. """ + if generator is not None and generator.device.type != self.device.type and self.device.type != "mps": + message = ( + f"The `generator` device is `{generator.device}` and does not match the pipeline " + f"device `{self.device}`, so the `generator` will be set to `None`. " + f"Please use `generator=torch.Generator(device=\"{self.device}\")` instead." + ) + deprecate( + "generator.device == 'cpu'", + "0.11.0", + message, + ) + generator = None + # Sample gaussian noise to begin loop image = torch.randn( (batch_size, self.unet.in_channels, self.unet.sample_size, self.unet.sample_size), generator=generator, + device=self.device, ) - image = image.to(self.device) # set step values self.scheduler.set_timesteps(num_inference_steps) - # Ignore use_clipped_model_output if the scheduler doesn't accept this argument - accepts_use_clipped_model_output = "use_clipped_model_output" in set( - inspect.signature(self.scheduler.step).parameters.keys() - ) - extra_kwargs = {} - if accepts_use_clipped_model_output: - extra_kwargs["use_clipped_model_output"] = use_clipped_model_output - for t in self.progress_bar(self.scheduler.timesteps): # 1. predict noise model_output model_output = self.unet(image, t).sample @@ -100,7 +105,9 @@ def __call__( # 2. predict previous mean of image x_t-1 and add variance depending on eta # eta corresponds to η in paper and should be between [0, 1] # do x_t -> x_t-1 - image = self.scheduler.step(model_output, t, image, eta, **extra_kwargs).prev_sample + image = self.scheduler.step( + model_output, t, image, eta=eta, use_clipped_model_output=use_clipped_model_output, generator=generator + ).prev_sample image = (image / 2 + 0.5).clamp(0, 1) image = image.cpu().permute(0, 2, 3, 1).numpy() diff --git a/src/diffusers/pipelines/ddpm/pipeline_ddpm.py b/src/diffusers/pipelines/ddpm/pipeline_ddpm.py index 3665c68efe1d..1de70776a1c1 100644 --- a/src/diffusers/pipelines/ddpm/pipeline_ddpm.py +++ b/src/diffusers/pipelines/ddpm/pipeline_ddpm.py @@ -80,12 +80,25 @@ def __call__( new_config["predict_epsilon"] = predict_epsilon self.scheduler._internal_dict = FrozenDict(new_config) + if generator is not None and generator.device.type != self.device.type and self.device.type != "mps": + message = ( + f"The `generator` device is `{generator.device}` and does not match the pipeline " + f"device `{self.device}`, so the `generator` will be set to `None`. " + f"Please use `torch.Generator(device=\"{self.device}\")` instead." + ) + deprecate( + "generator.device == 'cpu'", + "0.11.0", + message, + ) + generator = None + # Sample gaussian noise to begin loop image = torch.randn( (batch_size, self.unet.in_channels, self.unet.sample_size, self.unet.sample_size), generator=generator, + device=self.device, ) - image = image.to(self.device) # set step values self.scheduler.set_timesteps(num_inference_steps) diff --git a/src/diffusers/schedulers/scheduling_ddpm.py b/src/diffusers/schedulers/scheduling_ddpm.py index 08a73119e5b7..a19d91879cc3 100644 --- a/src/diffusers/schedulers/scheduling_ddpm.py +++ b/src/diffusers/schedulers/scheduling_ddpm.py @@ -292,10 +292,16 @@ def step( # 6. Add noise variance = 0 if t > 0: - noise = torch.randn( - model_output.size(), dtype=model_output.dtype, layout=model_output.layout, generator=generator - ).to(model_output.device) - variance = (self._get_variance(t, predicted_variance=predicted_variance) ** 0.5) * noise + device = model_output.device + if device.type == "mps": + # randn does not work reproducibly on mps + variance_noise = torch.randn(model_output.shape, dtype=model_output.dtype, generator=generator) + variance_noise = variance_noise.to(device) + else: + variance_noise = torch.randn( + model_output.shape, generator=generator, device=device, dtype=model_output.dtype + ) + variance = (self._get_variance(t, predicted_variance=predicted_variance) ** 0.5) * variance_noise pred_prev_sample = pred_prev_sample + variance diff --git a/tests/pipelines/ddim/test_ddim.py b/tests/pipelines/ddim/test_ddim.py index 4445fe7feecf..3c5134d9c603 100644 --- a/tests/pipelines/ddim/test_ddim.py +++ b/tests/pipelines/ddim/test_ddim.py @@ -54,16 +54,17 @@ def test_inference(self): if torch_device == "mps": _ = ddpm(num_inference_steps=1) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0] image_slice = image[0, -3:, -3:, -1] image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) + # TODO (Anton): update values in this PR expected_slice = np.array( [1.000e00, 5.717e-01, 4.717e-01, 1.000e00, 0.000e00, 1.000e00, 3.000e-04, 0.000e00, 9.000e-04] ) @@ -85,12 +86,13 @@ def test_inference_ema_bedroom(self): ddpm.to(torch_device) ddpm.set_progress_bar_config(disable=None) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, output_type="numpy").images image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 256, 256, 3) + # TODO (Anton): update values in this PR expected_slice = np.array([0.00605, 0.0201, 0.0344, 0.00235, 0.00185, 0.00025, 0.00215, 0.0, 0.00685]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 @@ -104,11 +106,12 @@ def test_inference_cifar10(self): ddim.to(torch_device) ddim.set_progress_bar_config(disable=None) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddim(generator=generator, eta=0.0, output_type="numpy").images image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) + # TODO (Anton): update values in this PR expected_slice = np.array([0.17235, 0.16175, 0.16005, 0.16255, 0.1497, 0.1513, 0.15045, 0.1442, 0.1453]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/tests/pipelines/ddpm/test_ddpm.py b/tests/pipelines/ddpm/test_ddpm.py index a09f77d12467..02c1534638b8 100644 --- a/tests/pipelines/ddpm/test_ddpm.py +++ b/tests/pipelines/ddpm/test_ddpm.py @@ -55,16 +55,17 @@ def test_inference(self): if torch_device == "mps": _ = ddpm(num_inference_steps=1) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0] image_slice = image[0, -3:, -3:, -1] image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) + # TODO (Anton): update values in this PR expected_slice = np.array( [5.589e-01, 7.089e-01, 2.632e-01, 6.841e-01, 1.000e-04, 9.999e-01, 1.973e-01, 1.000e-04, 8.010e-02] ) @@ -85,10 +86,10 @@ def test_inference_predict_epsilon(self): if torch_device == "mps": _ = ddpm(num_inference_steps=1) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image_eps = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", predict_epsilon=False)[0] image_slice = image[0, -3:, -3:, -1] @@ -112,11 +113,12 @@ def test_inference_cifar10(self): ddpm.to(torch_device) ddpm.set_progress_bar_config(disable=None) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, output_type="numpy").images image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) + # TODO (Anton): update values in this PR expected_slice = np.array([0.41995, 0.35885, 0.19385, 0.38475, 0.3382, 0.2647, 0.41545, 0.3582, 0.33845]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index da5bd3c24417..7e4ffac4656b 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -506,40 +506,8 @@ def test_output_format(self): assert isinstance(images, list) assert isinstance(images[0], PIL.Image.Image) - # Make sure the test passes for different values of random seed - @parameterized.expand([(0,), (4,)]) - def test_ddpm_ddim_equality(self, seed): - model_id = "google/ddpm-cifar10-32" - - unet = UNet2DModel.from_pretrained(model_id) - ddpm_scheduler = DDPMScheduler() - ddim_scheduler = DDIMScheduler() - - ddpm = DDPMPipeline(unet=unet, scheduler=ddpm_scheduler) - ddpm.to(torch_device) - ddpm.set_progress_bar_config(disable=None) - ddim = DDIMPipeline(unet=unet, scheduler=ddim_scheduler) - ddim.to(torch_device) - ddim.set_progress_bar_config(disable=None) - - generator = torch.manual_seed(seed) - ddpm_image = ddpm(generator=generator, output_type="numpy").images - - generator = torch.manual_seed(seed) - ddim_image = ddim( - generator=generator, - num_inference_steps=1000, - eta=1.0, - output_type="numpy", - use_clipped_model_output=True, # Need this to make DDIM match DDPM - ).images - - # the values aren't exactly equal, but the images look the same visually - assert np.abs(ddpm_image - ddim_image).max() < 1e-1 - - # Make sure the test passes for different values of random seed - @parameterized.expand([(0,), (4,)]) - def test_ddpm_ddim_equality_batched(self, seed): + def test_ddpm_ddim_equality_batched(self): + seed = 0 model_id = "google/ddpm-cifar10-32" unet = UNet2DModel.from_pretrained(model_id) @@ -554,12 +522,12 @@ def test_ddpm_ddim_equality_batched(self, seed): ddim.to(torch_device) ddim.set_progress_bar_config(disable=None) - generator = torch.manual_seed(seed) - ddpm_images = ddpm(batch_size=4, generator=generator, output_type="numpy").images + generator = torch.Generator().manual_seed(seed) + ddpm_images = ddpm(batch_size=2, generator=generator, output_type="numpy").images - generator = torch.manual_seed(seed) + generator = torch.Generator(device=torch_device).manual_seed(seed) ddim_images = ddim( - batch_size=4, + batch_size=2, generator=generator, num_inference_steps=1000, eta=1.0, From 705853922fc6543e52d55bf619afdb6e0e5a2b11 Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 18:10:16 +0100 Subject: [PATCH 2/9] style --- src/diffusers/pipelines/ddim/pipeline_ddim.py | 2 +- src/diffusers/pipelines/ddpm/pipeline_ddpm.py | 2 +- tests/test_pipelines.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/ddim/pipeline_ddim.py b/src/diffusers/pipelines/ddim/pipeline_ddim.py index 2610f9d3f52a..d0bca8038ec4 100644 --- a/src/diffusers/pipelines/ddim/pipeline_ddim.py +++ b/src/diffusers/pipelines/ddim/pipeline_ddim.py @@ -79,7 +79,7 @@ def __call__( message = ( f"The `generator` device is `{generator.device}` and does not match the pipeline " f"device `{self.device}`, so the `generator` will be set to `None`. " - f"Please use `generator=torch.Generator(device=\"{self.device}\")` instead." + f'Please use `generator=torch.Generator(device="{self.device}")` instead.' ) deprecate( "generator.device == 'cpu'", diff --git a/src/diffusers/pipelines/ddpm/pipeline_ddpm.py b/src/diffusers/pipelines/ddpm/pipeline_ddpm.py index 1de70776a1c1..d145c5d518a1 100644 --- a/src/diffusers/pipelines/ddpm/pipeline_ddpm.py +++ b/src/diffusers/pipelines/ddpm/pipeline_ddpm.py @@ -84,7 +84,7 @@ def __call__( message = ( f"The `generator` device is `{generator.device}` and does not match the pipeline " f"device `{self.device}`, so the `generator` will be set to `None`. " - f"Please use `torch.Generator(device=\"{self.device}\")` instead." + f'Please use `torch.Generator(device="{self.device}")` instead.' ) deprecate( "generator.device == 'cpu'", diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 7e4ffac4656b..db2d9bf1b338 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -522,7 +522,7 @@ def test_ddpm_ddim_equality_batched(self): ddim.to(torch_device) ddim.set_progress_bar_config(disable=None) - generator = torch.Generator().manual_seed(seed) + generator = torch.Generator(device=torch_device).manual_seed(seed) ddpm_images = ddpm(batch_size=2, generator=generator, output_type="numpy").images generator = torch.Generator(device=torch_device).manual_seed(seed) From c4734e01714386f549503183e2c45ed33840aa17 Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 18:12:27 +0100 Subject: [PATCH 3/9] fix --- tests/test_pipelines.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index db2d9bf1b338..b9685d41f828 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -42,7 +42,6 @@ from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME from diffusers.utils import CONFIG_NAME, WEIGHTS_NAME, floats_tensor, slow, torch_device from diffusers.utils.testing_utils import CaptureLogger, get_tests_dir, require_torch_gpu -from parameterized import parameterized from PIL import Image from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTextConfig, CLIPTextModel, CLIPTokenizer From b10c793f249c1b1fec6375818328b1a5fcbb02fe Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 21:36:35 +0100 Subject: [PATCH 4/9] update values --- .../train_unconditional.py | 9 +++++++-- tests/pipelines/ddim/test_ddim.py | 13 +++++-------- tests/pipelines/ddpm/test_ddpm.py | 10 ++++------ .../test_latent_diffusion_superresolution.py | 2 +- tests/test_pipelines.py | 14 ++++++++++---- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/examples/unconditional_image_generation/train_unconditional.py b/examples/unconditional_image_generation/train_unconditional.py index 0eadecbd3095..54f6013567be 100644 --- a/examples/unconditional_image_generation/train_unconditional.py +++ b/examples/unconditional_image_generation/train_unconditional.py @@ -11,10 +11,11 @@ from accelerate import Accelerator from accelerate.logging import get_logger from datasets import load_dataset -from diffusers import DDPMPipeline, DDPMScheduler, UNet2DModel +from diffusers import DDPMPipeline, DDPMScheduler, UNet2DModel, __version__ from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel from huggingface_hub import HfFolder, Repository, whoami +from packaging import version from torchvision.transforms import ( CenterCrop, Compose, @@ -28,6 +29,7 @@ logger = get_logger(__name__) +diffusers_version = version.parse(version.parse(__version__).base_version) def _extract_into_tensor(arr, timesteps, broadcast_shape): @@ -406,7 +408,10 @@ def transforms(examples): scheduler=noise_scheduler, ) - generator = torch.manual_seed(0) + if diffusers_version < version.parse("0.8.0"): + generator = torch.manual_seed(0) + else: + generator = torch.Generator(device=pipeline.device).manual_seed(0) # run pipeline in inference (sample random noise and denoise) images = pipeline( generator=generator, diff --git a/tests/pipelines/ddim/test_ddim.py b/tests/pipelines/ddim/test_ddim.py index 3c5134d9c603..7edbedf68de8 100644 --- a/tests/pipelines/ddim/test_ddim.py +++ b/tests/pipelines/ddim/test_ddim.py @@ -19,7 +19,7 @@ import torch from diffusers import DDIMPipeline, DDIMScheduler, UNet2DModel -from diffusers.utils.testing_utils import require_torch, slow, torch_device +from diffusers.utils.testing_utils import require_torch_gpu, slow, torch_device from ...test_pipelines_common import PipelineTesterMixin @@ -64,9 +64,8 @@ def test_inference(self): image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) - # TODO (Anton): update values in this PR expected_slice = np.array( - [1.000e00, 5.717e-01, 4.717e-01, 1.000e00, 0.000e00, 1.000e00, 3.000e-04, 0.000e00, 9.000e-04] + [8.543e-04, 9.880e-01, 3.983e-01, 7.988e-04, 9.566e-02, 1.200e-03, 4.849e-04, 1.000e00, 9.997e-01] ) tolerance = 1e-2 if torch_device != "mps" else 3e-2 assert np.abs(image_slice.flatten() - expected_slice).max() < tolerance @@ -74,7 +73,7 @@ def test_inference(self): @slow -@require_torch +@require_torch_gpu class DDIMPipelineIntegrationTests(unittest.TestCase): def test_inference_ema_bedroom(self): model_id = "google/ddpm-ema-bedroom-256" @@ -92,8 +91,7 @@ def test_inference_ema_bedroom(self): image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 256, 256, 3) - # TODO (Anton): update values in this PR - expected_slice = np.array([0.00605, 0.0201, 0.0344, 0.00235, 0.00185, 0.00025, 0.00215, 0.0, 0.00685]) + expected_slice = np.array([0.4215, 0.2433, 0.2897, 0.4166, 0.2351, 0.2893, 0.4241, 0.2337, 0.3011]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 def test_inference_cifar10(self): @@ -112,6 +110,5 @@ def test_inference_cifar10(self): image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) - # TODO (Anton): update values in this PR - expected_slice = np.array([0.17235, 0.16175, 0.16005, 0.16255, 0.1497, 0.1513, 0.15045, 0.1442, 0.1453]) + expected_slice = np.array([0.2060, 0.2042, 0.2022, 0.2193, 0.2146, 0.2110, 0.2471, 0.2446, 0.2388]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/tests/pipelines/ddpm/test_ddpm.py b/tests/pipelines/ddpm/test_ddpm.py index 02c1534638b8..35aa2d404efb 100644 --- a/tests/pipelines/ddpm/test_ddpm.py +++ b/tests/pipelines/ddpm/test_ddpm.py @@ -20,7 +20,7 @@ from diffusers import DDPMPipeline, DDPMScheduler, UNet2DModel from diffusers.utils import deprecate -from diffusers.utils.testing_utils import require_torch, slow, torch_device +from diffusers.utils.testing_utils import require_torch_gpu, slow, torch_device from ...test_pipelines_common import PipelineTesterMixin @@ -65,9 +65,8 @@ def test_inference(self): image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) - # TODO (Anton): update values in this PR expected_slice = np.array( - [5.589e-01, 7.089e-01, 2.632e-01, 6.841e-01, 1.000e-04, 9.999e-01, 1.973e-01, 1.000e-04, 8.010e-02] + [8.296e-05, 6.484e-01, 5.530e-01, 7.004e-02, 4.530e-01, 2.200e-01, 2.348e-01, 9.999e-01, 7.396e-01] ) tolerance = 1e-2 if torch_device != "mps" else 3e-2 assert np.abs(image_slice.flatten() - expected_slice).max() < tolerance @@ -101,7 +100,7 @@ def test_inference_predict_epsilon(self): @slow -@require_torch +@require_torch_gpu class DDPMPipelineIntegrationTests(unittest.TestCase): def test_inference_cifar10(self): model_id = "google/ddpm-cifar10-32" @@ -119,6 +118,5 @@ def test_inference_cifar10(self): image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 32, 32, 3) - # TODO (Anton): update values in this PR - expected_slice = np.array([0.41995, 0.35885, 0.19385, 0.38475, 0.3382, 0.2647, 0.41545, 0.3582, 0.33845]) + expected_slice = np.array([0.4454, 0.2025, 0.0315, 0.3023, 0.2575, 0.1031, 0.0953, 0.1604, 0.2020]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py b/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py index f5ec56d1bdc2..7b22de3201f5 100644 --- a/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py +++ b/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py @@ -83,7 +83,7 @@ def test_inference_superresolution(self): generator = torch.manual_seed(0) _ = ldm(init_image, generator=generator, num_inference_steps=1, output_type="numpy").images - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ldm(init_image, generator=generator, num_inference_steps=2, output_type="numpy").images image_slice = image[0, -3:, -3:, -1] diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index b9685d41f828..9fd491c8459e 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -92,10 +92,12 @@ def test_download_no_safety_checker(self): pipe = StableDiffusionPipeline.from_pretrained( "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None ) + pipe = pipe.to(torch_device) generator = torch.Generator(device=torch_device).manual_seed(0) out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images pipe_2 = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + pipe_2 = pipe_2.to(torch_device) generator_2 = torch.Generator(device=torch_device).manual_seed(0) out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator_2, output_type="numpy").images @@ -106,12 +108,14 @@ def test_load_no_safety_checker_explicit_locally(self): pipe = StableDiffusionPipeline.from_pretrained( "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None ) + pipe = pipe.to(torch_device) generator = torch.Generator(device=torch_device).manual_seed(0) out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images with tempfile.TemporaryDirectory() as tmpdirname: pipe.save_pretrained(tmpdirname) pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname, safety_checker=None) + pipe_2 = pipe_2.to(torch_device) generator_2 = torch.Generator(device=torch_device).manual_seed(0) out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator_2, output_type="numpy").images @@ -120,12 +124,14 @@ def test_load_no_safety_checker_explicit_locally(self): def test_load_no_safety_checker_default_locally(self): prompt = "hello" pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + pipe = pipe.to(torch_device) generator = torch.Generator(device=torch_device).manual_seed(0) out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images with tempfile.TemporaryDirectory() as tmpdirname: pipe.save_pretrained(tmpdirname) pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname) + pipe_2 = pipe_2.to(torch_device) generator_2 = torch.Generator(device=torch_device).manual_seed(0) out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator_2, output_type="numpy").images @@ -430,7 +436,7 @@ def test_from_pretrained_save_pretrained(self): new_ddpm = DDPMPipeline.from_pretrained(tmpdirname) new_ddpm.to(torch_device) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, output_type="numpy").images generator = generator.manual_seed(0) @@ -451,7 +457,7 @@ def test_from_pretrained_hub(self): ddpm_from_hub = ddpm_from_hub.to(torch_device) ddpm_from_hub.set_progress_bar_config(disable=None) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm(generator=generator, output_type="numpy").images generator = generator.manual_seed(0) @@ -474,7 +480,7 @@ def test_from_pretrained_hub_pass_model(self): ddpm_from_hub = ddpm_from_hub.to(torch_device) ddpm_from_hub_custom_model.set_progress_bar_config(disable=None) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) image = ddpm_from_hub_custom_model(generator=generator, output_type="numpy").images generator = generator.manual_seed(0) @@ -490,7 +496,7 @@ def test_output_format(self): pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) - generator = torch.manual_seed(0) + generator = torch.Generator(device=torch_device).manual_seed(0) images = pipe(generator=generator, output_type="numpy").images assert images.shape == (1, 32, 32, 3) assert isinstance(images, np.ndarray) From c8744bee9d713d2c50c5a95ada24106e2b793bf5 Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 21:57:06 +0100 Subject: [PATCH 5/9] fix fast tests --- .../train_unconditional.py | 1 + tests/pipelines/ddim/test_ddim.py | 18 +++++++----------- tests/pipelines/ddpm/test_ddpm.py | 18 +++++++----------- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/examples/unconditional_image_generation/train_unconditional.py b/examples/unconditional_image_generation/train_unconditional.py index 54f6013567be..50edffa1ebcc 100644 --- a/examples/unconditional_image_generation/train_unconditional.py +++ b/examples/unconditional_image_generation/train_unconditional.py @@ -408,6 +408,7 @@ def transforms(examples): scheduler=noise_scheduler, ) + deprecate("todo: remove this check", "0.10.0", "when the most used version is >= 0.8.0") if diffusers_version < version.parse("0.8.0"): generator = torch.manual_seed(0) else: diff --git a/tests/pipelines/ddim/test_ddim.py b/tests/pipelines/ddim/test_ddim.py index 7edbedf68de8..68cf1445beab 100644 --- a/tests/pipelines/ddim/test_ddim.py +++ b/tests/pipelines/ddim/test_ddim.py @@ -43,21 +43,18 @@ def dummy_uncond_unet(self): return model def test_inference(self): + device = "cpu" unet = self.dummy_uncond_unet scheduler = DDIMScheduler() ddpm = DDIMPipeline(unet=unet, scheduler=scheduler) - ddpm.to(torch_device) + ddpm.to(device) ddpm.set_progress_bar_config(disable=None) - # Warmup pass when using mps (see #372) - if torch_device == "mps": - _ = ddpm(num_inference_steps=1) - - generator = torch.Generator(device=torch_device).manual_seed(0) + generator = torch.Generator(device=device).manual_seed(0) image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images - generator = torch.Generator(device=torch_device).manual_seed(0) + generator = torch.Generator(device=device).manual_seed(0) image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0] image_slice = image[0, -3:, -3:, -1] @@ -65,11 +62,10 @@ def test_inference(self): assert image.shape == (1, 32, 32, 3) expected_slice = np.array( - [8.543e-04, 9.880e-01, 3.983e-01, 7.988e-04, 9.566e-02, 1.200e-03, 4.849e-04, 1.000e00, 9.997e-01] + [1.000e00, 5.717e-01, 4.717e-01, 1.000e00, 0.000e00, 1.000e00, 3.000e-04, 0.000e00, 9.000e-04] ) - tolerance = 1e-2 if torch_device != "mps" else 3e-2 - assert np.abs(image_slice.flatten() - expected_slice).max() < tolerance - assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < tolerance + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 @slow diff --git a/tests/pipelines/ddpm/test_ddpm.py b/tests/pipelines/ddpm/test_ddpm.py index 35aa2d404efb..e16e0d6e8cbd 100644 --- a/tests/pipelines/ddpm/test_ddpm.py +++ b/tests/pipelines/ddpm/test_ddpm.py @@ -44,21 +44,18 @@ def dummy_uncond_unet(self): return model def test_inference(self): + device = "cpu" unet = self.dummy_uncond_unet scheduler = DDPMScheduler() ddpm = DDPMPipeline(unet=unet, scheduler=scheduler) - ddpm.to(torch_device) + ddpm.to(device) ddpm.set_progress_bar_config(disable=None) - # Warmup pass when using mps (see #372) - if torch_device == "mps": - _ = ddpm(num_inference_steps=1) - - generator = torch.Generator(device=torch_device).manual_seed(0) + generator = torch.Generator(device=device).manual_seed(0) image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images - generator = torch.Generator(device=torch_device).manual_seed(0) + generator = torch.Generator(device=device).manual_seed(0) image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0] image_slice = image[0, -3:, -3:, -1] @@ -66,11 +63,10 @@ def test_inference(self): assert image.shape == (1, 32, 32, 3) expected_slice = np.array( - [8.296e-05, 6.484e-01, 5.530e-01, 7.004e-02, 4.530e-01, 2.200e-01, 2.348e-01, 9.999e-01, 7.396e-01] + [5.589e-01, 7.089e-01, 2.632e-01, 6.841e-01, 1.000e-04, 9.999e-01, 1.973e-01, 1.000e-04, 8.010e-02] ) - tolerance = 1e-2 if torch_device != "mps" else 3e-2 - assert np.abs(image_slice.flatten() - expected_slice).max() < tolerance - assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < tolerance + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 def test_inference_predict_epsilon(self): deprecate("remove this test", "0.10.0", "remove") From 853ccc2eac290a97776bf81a5053ab7119e94ca8 Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 21:59:23 +0100 Subject: [PATCH 6/9] trigger slow tests --- .github/workflows/push_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 2beb05e8eaca..793563a8d582 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -4,6 +4,9 @@ on: push: branches: - main + pull_request: + branches: + - main env: DIFFUSERS_IS_CI: yes From 2d98c19f40ac5e487562653445a67459ff7d3893 Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 22:00:25 +0100 Subject: [PATCH 7/9] deprecate --- examples/unconditional_image_generation/train_unconditional.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/unconditional_image_generation/train_unconditional.py b/examples/unconditional_image_generation/train_unconditional.py index 50edffa1ebcc..54a94d98b578 100644 --- a/examples/unconditional_image_generation/train_unconditional.py +++ b/examples/unconditional_image_generation/train_unconditional.py @@ -14,6 +14,7 @@ from diffusers import DDPMPipeline, DDPMScheduler, UNet2DModel, __version__ from diffusers.optimization import get_scheduler from diffusers.training_utils import EMAModel +from diffusers.utils import deprecate from huggingface_hub import HfFolder, Repository, whoami from packaging import version from torchvision.transforms import ( From 07b479f9adbd606218b91de6f918e07444937ffa Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 22:44:44 +0100 Subject: [PATCH 8/9] last value fixes --- .github/workflows/push_tests.yml | 3 --- tests/pipelines/ddim/test_ddim.py | 2 +- .../test_latent_diffusion_superresolution.py | 17 ++++++----------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 793563a8d582..2beb05e8eaca 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -4,9 +4,6 @@ on: push: branches: - main - pull_request: - branches: - - main env: DIFFUSERS_IS_CI: yes diff --git a/tests/pipelines/ddim/test_ddim.py b/tests/pipelines/ddim/test_ddim.py index 68cf1445beab..81c49912be5c 100644 --- a/tests/pipelines/ddim/test_ddim.py +++ b/tests/pipelines/ddim/test_ddim.py @@ -87,7 +87,7 @@ def test_inference_ema_bedroom(self): image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 256, 256, 3) - expected_slice = np.array([0.4215, 0.2433, 0.2897, 0.4166, 0.2351, 0.2893, 0.4241, 0.2337, 0.3011]) + expected_slice = np.array([0.1546, 0.1561, 0.1595, 0.1564, 0.1569, 0.1585, 0.1554, 0.1550, 0.1575]) assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 def test_inference_cifar10(self): diff --git a/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py b/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py index 7b22de3201f5..f402d2f2a70e 100644 --- a/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py +++ b/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py @@ -68,30 +68,25 @@ def dummy_vq_model(self): return model def test_inference_superresolution(self): + device = "cpu" unet = self.dummy_uncond_unet scheduler = DDIMScheduler() vqvae = self.dummy_vq_model ldm = LDMSuperResolutionPipeline(unet=unet, vqvae=vqvae, scheduler=scheduler) - ldm.to(torch_device) + ldm.to(device) ldm.set_progress_bar_config(disable=None) - init_image = self.dummy_image.to(torch_device) - - # Warmup pass when using mps (see #372) - if torch_device == "mps": - generator = torch.manual_seed(0) - _ = ldm(init_image, generator=generator, num_inference_steps=1, output_type="numpy").images + init_image = self.dummy_image.to(device) - generator = torch.Generator(device=torch_device).manual_seed(0) + generator = torch.Generator(device=device).manual_seed(0) image = ldm(init_image, generator=generator, num_inference_steps=2, output_type="numpy").images image_slice = image[0, -3:, -3:, -1] assert image.shape == (1, 64, 64, 3) - expected_slice = np.array([0.8634, 0.8186, 0.6416, 0.6846, 0.4427, 0.5676, 0.4679, 0.6247, 0.5176]) - tolerance = 1e-2 if torch_device != "mps" else 3e-2 - assert np.abs(image_slice.flatten() - expected_slice).max() < tolerance + expected_slice = np.array([0.8678, 0.8245, 0.6381, 0.6830, 0.4385, 0.5599, 0.4641, 0.6201, 0.5150]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 @slow From c72dea2216a59e7480412cec6b70bf76212467ef Mon Sep 17 00:00:00 2001 From: anton-l Date: Wed, 9 Nov 2022 22:55:33 +0100 Subject: [PATCH 9/9] mps fixes --- tests/test_pipelines.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 9fd491c8459e..775ab689bdcb 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -93,12 +93,16 @@ def test_download_no_safety_checker(self): "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None ) pipe = pipe.to(torch_device) - generator = torch.Generator(device=torch_device).manual_seed(0) + if torch_device == "mps": + # device type MPS is not supported for torch.Generator() api. + generator = torch.manual_seed(0) + else: + generator = torch.Generator(device=torch_device).manual_seed(0) out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images pipe_2 = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") pipe_2 = pipe_2.to(torch_device) - generator_2 = torch.Generator(device=torch_device).manual_seed(0) + generator_2 = generator.manual_seed(0) out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator_2, output_type="numpy").images assert np.max(np.abs(out - out_2)) < 1e-3 @@ -109,14 +113,18 @@ def test_load_no_safety_checker_explicit_locally(self): "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None ) pipe = pipe.to(torch_device) - generator = torch.Generator(device=torch_device).manual_seed(0) + if torch_device == "mps": + # device type MPS is not supported for torch.Generator() api. + generator = torch.manual_seed(0) + else: + generator = torch.Generator(device=torch_device).manual_seed(0) out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images with tempfile.TemporaryDirectory() as tmpdirname: pipe.save_pretrained(tmpdirname) pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname, safety_checker=None) pipe_2 = pipe_2.to(torch_device) - generator_2 = torch.Generator(device=torch_device).manual_seed(0) + generator_2 = generator.manual_seed(0) out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator_2, output_type="numpy").images assert np.max(np.abs(out - out_2)) < 1e-3 @@ -125,14 +133,18 @@ def test_load_no_safety_checker_default_locally(self): prompt = "hello" pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") pipe = pipe.to(torch_device) - generator = torch.Generator(device=torch_device).manual_seed(0) + if torch_device == "mps": + # device type MPS is not supported for torch.Generator() api. + generator = torch.manual_seed(0) + else: + generator = torch.Generator(device=torch_device).manual_seed(0) out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images with tempfile.TemporaryDirectory() as tmpdirname: pipe.save_pretrained(tmpdirname) pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname) pipe_2 = pipe_2.to(torch_device) - generator_2 = torch.Generator(device=torch_device).manual_seed(0) + generator_2 = generator.manual_seed(0) out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator_2, output_type="numpy").images assert np.max(np.abs(out - out_2)) < 1e-3