From b06823e1b9146f9996127b2cfddb18d2d5647da8 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 12 Jun 2023 10:43:39 +0000 Subject: [PATCH 1/5] Correct timestep inpaint --- .../pipelines/controlnet/pipeline_controlnet_inpaint.py | 3 ++- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index 5ce2fd5543b8..d00dd3335eed 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -1306,7 +1306,8 @@ def __call__( init_mask = mask[:1] if i < len(timesteps) - 1: - init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([t])) + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([noise_timestep])) latents = (1 - init_mask) * init_latents_proper + init_mask * latents diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index b07a5555f1c7..d29d71488163 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -1038,7 +1038,8 @@ def __call__( init_mask = mask[:1] if i < len(timesteps) - 1: - init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([t])) + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([noise_timestep])) latents = (1 - init_mask) * init_latents_proper + init_mask * latents From 888b96d0b79fa04f9ef206db609793647773b0d2 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 12 Jun 2023 11:43:25 +0000 Subject: [PATCH 2/5] make style --- .../pipelines/controlnet/pipeline_controlnet.py | 2 +- .../controlnet/pipeline_controlnet_inpaint.py | 17 +++++++++++------ .../pipeline_stable_diffusion_inpaint.py | 4 +++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py index 89398b6f01f9..2a86ee0dfe1e 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -611,7 +611,7 @@ def check_image(self, image, prompt, prompt_embeds): and not image_is_np_list ): raise TypeError( - "image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors" + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" ) if image_is_pil: diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index d00dd3335eed..b7a2b7e1c007 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -770,7 +770,7 @@ def check_image(self, image, prompt, prompt_embeds): and not image_is_np_list ): raise TypeError( - "image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors" + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" ) if image_is_pil: @@ -1303,13 +1303,18 @@ def __call__( if num_channels_unet == 4: init_latents_proper = image_latents[:1] - init_mask = mask[:1] + mask[:1] if i < len(timesteps) - 1: - noise_timestep = timesteps[i + 1] - init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([noise_timestep])) - - latents = (1 - init_mask) * init_latents_proper + init_mask * latents + # noise_timestep = timesteps[i + 1] + noise_timestep = timesteps[i] + print(noise_timestep) + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) + + # latents = (1 - init_mask) * init_latents_proper + init_mask * latents + latents = init_latents_proper # call the callback, if provided if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index d29d71488163..d958f0e3fb72 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -1039,7 +1039,9 @@ def __call__( if i < len(timesteps) - 1: noise_timestep = timesteps[i + 1] - init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([noise_timestep])) + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) latents = (1 - init_mask) * init_latents_proper + init_mask * latents From aa8e39d4584409bacbe03757e0448364abea0b7c Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 12 Jun 2023 11:47:53 +0000 Subject: [PATCH 3/5] Fix --- .../pipelines/controlnet/pipeline_controlnet_inpaint.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index b7a2b7e1c007..80bb20a6d4bf 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -1306,15 +1306,12 @@ def __call__( mask[:1] if i < len(timesteps) - 1: - # noise_timestep = timesteps[i + 1] - noise_timestep = timesteps[i] - print(noise_timestep) + noise_timestep = timesteps[i + 1] init_latents_proper = self.scheduler.add_noise( init_latents_proper, noise, torch.tensor([noise_timestep]) ) - # latents = (1 - init_mask) * init_latents_proper + init_mask * latents - latents = init_latents_proper + latents = (1 - init_mask) * init_latents_proper + init_mask * latents # call the callback, if provided if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): From cf1cb84c961befe504534b310de1fa497627c605 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 12 Jun 2023 13:48:39 +0200 Subject: [PATCH 4/5] Apply suggestions from code review --- .../pipelines/controlnet/pipeline_controlnet_inpaint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index 80bb20a6d4bf..165e2d88dca6 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -1303,7 +1303,7 @@ def __call__( if num_channels_unet == 4: init_latents_proper = image_latents[:1] - mask[:1] + init_mask = mask[:1] if i < len(timesteps) - 1: noise_timestep = timesteps[i + 1] From ff0693c2b30734588c225a7d0cd165d2f9fca0de Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 12 Jun 2023 11:49:02 +0000 Subject: [PATCH 5/5] make style --- .../pipelines/controlnet/pipeline_controlnet_img2img.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py index 0e984d8ae5e3..e42b27958446 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -638,7 +638,7 @@ def check_image(self, image, prompt, prompt_embeds): and not image_is_np_list ): raise TypeError( - "image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors" + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" ) if image_is_pil: