From cd87884f78d03c5e3aab2cc940a0774a83dea04a Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 12 Jun 2023 13:59:38 +0200 Subject: [PATCH] [Stable Diffusion Inpaint & ControlNet inpaint] Correct timestep inpaint (#3749) * Correct timestep inpaint * make style * Fix * Apply suggestions from code review * make style --- src/diffusers/pipelines/controlnet/pipeline_controlnet.py | 2 +- .../pipelines/controlnet/pipeline_controlnet_img2img.py | 2 +- .../pipelines/controlnet/pipeline_controlnet_inpaint.py | 7 +++++-- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 5 ++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py index 89398b6f01..2a86ee0dfe 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -611,7 +611,7 @@ class StableDiffusionControlNetPipeline(DiffusionPipeline, TextualInversionLoade and not image_is_np_list ): raise TypeError( - "image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors" + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" ) if image_is_pil: diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py index 0e984d8ae5..e42b279584 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -638,7 +638,7 @@ class StableDiffusionControlNetImg2ImgPipeline(DiffusionPipeline, TextualInversi and not image_is_np_list ): raise TypeError( - "image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors" + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" ) if image_is_pil: diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index 5ce2fd5543..165e2d88dc 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -770,7 +770,7 @@ class StableDiffusionControlNetInpaintPipeline(DiffusionPipeline, TextualInversi and not image_is_np_list ): raise TypeError( - "image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors" + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" ) if image_is_pil: @@ -1306,7 +1306,10 @@ class StableDiffusionControlNetInpaintPipeline(DiffusionPipeline, TextualInversi init_mask = mask[:1] if i < len(timesteps) - 1: - init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([t])) + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) latents = (1 - init_mask) * init_latents_proper + init_mask * latents diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index b07a5555f1..d958f0e3fb 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -1038,7 +1038,10 @@ class StableDiffusionInpaintPipeline(DiffusionPipeline, TextualInversionLoaderMi init_mask = mask[:1] if i < len(timesteps) - 1: - init_latents_proper = self.scheduler.add_noise(init_latents_proper, noise, torch.tensor([t])) + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) latents = (1 - init_mask) * init_latents_proper + init_mask * latents