From ef3844d3a83583f36d0166be6753d062b3cbd7dc Mon Sep 17 00:00:00 2001 From: Hans Brouwer Date: Wed, 21 Jun 2023 13:11:45 +0200 Subject: [PATCH] Support ControlNet models with different number of channels in control images (#3815) support ControlNet models with a different hint_channels value (e.g. TemporalNet2) --- src/diffusers/models/controlnet.py | 2 ++ src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/diffusers/models/controlnet.py b/src/diffusers/models/controlnet.py index 0b0ce0be54..88562dd371 100644 --- a/src/diffusers/models/controlnet.py +++ b/src/diffusers/models/controlnet.py @@ -93,6 +93,7 @@ class ControlNetModel(ModelMixin, ConfigMixin): def __init__( self, in_channels: int = 4, + conditioning_channels: int = 3, flip_sin_to_cos: bool = True, freq_shift: int = 0, down_block_types: Tuple[str] = ( @@ -185,6 +186,7 @@ class ControlNetModel(ModelMixin, ConfigMixin): self.controlnet_cond_embedding = ControlNetConditioningEmbedding( conditioning_embedding_channels=block_out_channels[0], block_out_channels=conditioning_embedding_out_channels, + conditioning_channels=conditioning_channels, ) self.down_blocks = nn.ModuleList([]) diff --git a/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py b/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py index e59b91e486..746844ea1e 100644 --- a/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +++ b/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py @@ -286,6 +286,7 @@ def create_unet_diffusers_config(original_config, image_size: int, controlnet=Fa "use_linear_projection": use_linear_projection, "class_embed_type": class_embed_type, "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "conditioning_channels": unet_params.hint_channels, } if not controlnet: