From 5eead70b93a26124eef9135a93055752e432a345 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 15:53:35 +0530 Subject: [PATCH 1/9] debugging --- src/diffusers/models/unet_2d_condition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 5695429489..c2bf73a2fc 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -966,6 +966,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) encoder_attention_mask=encoder_attention_mask, ) # To support T2I-Adapter-XL + print(f"From UNet: {len(down_block_additional_residuals)}") if is_adapter and len(down_block_additional_residuals) > 0: sample += down_block_additional_residuals.pop(0) From d44c0b01825d199235851ef425c706b5ce415435 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 15:58:51 +0530 Subject: [PATCH 2/9] debugging --- src/diffusers/models/unet_2d_condition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index c2bf73a2fc..48a04bd9d1 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -920,6 +920,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) is_adapter = mid_block_additional_residual is None and down_block_additional_residuals is not None down_block_res_samples = (sample,) + print(f"From UNet: {len(down_block_additional_residuals)}") for downsample_block in self.down_blocks: if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: # For t2i-adapter CrossAttnDownBlock2D From a893a4323179eb44ffb077a4cee9a7b4df622329 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 16:00:46 +0530 Subject: [PATCH 3/9] debugging --- src/diffusers/models/adapter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/diffusers/models/adapter.py b/src/diffusers/models/adapter.py index fabe5fd94e..5b0d0f8793 100644 --- a/src/diffusers/models/adapter.py +++ b/src/diffusers/models/adapter.py @@ -136,6 +136,7 @@ class FullAdapter(nn.Module): downscale_factor: int = 8, ): super().__init__() + print(f"From {self.__class__.name} channels: {channels}.") in_channels = in_channels * downscale_factor**2 @@ -265,6 +266,7 @@ class LightAdapter(nn.Module): downscale_factor: int = 8, ): super().__init__() + print(f"From {self.__class__.name} channels: {channels}.") in_channels = in_channels * downscale_factor**2 From 6c5d3766d7c2e7e4bd6229ce2b9a459bb985c108 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 16:01:35 +0530 Subject: [PATCH 4/9] debugging --- src/diffusers/models/adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/models/adapter.py b/src/diffusers/models/adapter.py index 5b0d0f8793..40627b965b 100644 --- a/src/diffusers/models/adapter.py +++ b/src/diffusers/models/adapter.py @@ -136,7 +136,7 @@ class FullAdapter(nn.Module): downscale_factor: int = 8, ): super().__init__() - print(f"From {self.__class__.name} channels: {channels}.") + print(f"From {self.__class__} channels: {channels}.") in_channels = in_channels * downscale_factor**2 @@ -266,7 +266,7 @@ class LightAdapter(nn.Module): downscale_factor: int = 8, ): super().__init__() - print(f"From {self.__class__.name} channels: {channels}.") + print(f"From {self.__class__} channels: {channels}.") in_channels = in_channels * downscale_factor**2 From 99788b247ffdd5652f822dbff934928236fad7a4 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 16:13:34 +0530 Subject: [PATCH 5/9] debugging --- src/diffusers/models/unet_2d_condition.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 48a04bd9d1..f041bf56ae 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -920,7 +920,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) is_adapter = mid_block_additional_residual is None and down_block_additional_residuals is not None down_block_res_samples = (sample,) - print(f"From UNet: {len(down_block_additional_residuals)}") + print(f"From UNet before down blocks: {len(down_block_additional_residuals)}") for downsample_block in self.down_blocks: if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: # For t2i-adapter CrossAttnDownBlock2D @@ -967,7 +967,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) encoder_attention_mask=encoder_attention_mask, ) # To support T2I-Adapter-XL - print(f"From UNet: {len(down_block_additional_residuals)}") + print(f"From UNet in mid block: {len(down_block_additional_residuals)}") if is_adapter and len(down_block_additional_residuals) > 0: sample += down_block_additional_residuals.pop(0) From 5c98d93767638deab562fe575baa63c204279782 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 16:19:56 +0530 Subject: [PATCH 6/9] debugging --- .../pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py index 4aa911198a..a3442a0a87 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -728,6 +728,7 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline): latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) # predict the noise residual + print(f"From pipeline: {len(adapter_state)}.") noise_pred = self.unet( latent_model_input, t, From 66955beba62209fca5fcf3ef51250d90bf57634b Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 16:25:35 +0530 Subject: [PATCH 7/9] debugging --- .../pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py index a3442a0a87..3f7e19c754 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -711,6 +711,7 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline): # 7. Denoising loop adapter_state = self.adapter(adapter_input) + print(f"From pipeline (before rejigging): {len(adapter_state)}.") for k, v in enumerate(adapter_state): adapter_state[k] = v * adapter_conditioning_scale if num_images_per_prompt > 1: @@ -719,6 +720,7 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline): if do_classifier_free_guidance: for k, v in enumerate(adapter_state): adapter_state[k] = torch.cat([v] * 2, dim=0) + print(f"From pipeline (after rejigging): {len(adapter_state)}.") num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order with self.progress_bar(total=num_inference_steps) as progress_bar: @@ -728,7 +730,6 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline): latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) # predict the noise residual - print(f"From pipeline: {len(adapter_state)}.") noise_pred = self.unet( latent_model_input, t, From 54e683f77397a8115a5d20bc885c8353ee07588e Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 16:30:51 +0530 Subject: [PATCH 8/9] debugging --- src/diffusers/models/adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/models/adapter.py b/src/diffusers/models/adapter.py index 40627b965b..bc0c58dc29 100644 --- a/src/diffusers/models/adapter.py +++ b/src/diffusers/models/adapter.py @@ -164,7 +164,7 @@ class FullAdapter(nn.Module): for block in self.body: x = block(x) features.append(x) - + print(f"Number of features: {len(features)}") return features @@ -293,7 +293,7 @@ class LightAdapter(nn.Module): for block in self.body: x = block(x) features.append(x) - + print(f"Number of features: {len(features)}") return features From 52bc39c997a3ec29df22946964c071d3c0bad830 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 23 Aug 2023 17:01:25 +0530 Subject: [PATCH 9/9] revert prints. --- src/diffusers/models/adapter.py | 5 ++--- src/diffusers/models/unet_2d_condition.py | 2 -- .../t2i_adapter/pipeline_stable_diffusion_adapter.py | 2 -- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/diffusers/models/adapter.py b/src/diffusers/models/adapter.py index bc0c58dc29..0a79995cb4 100644 --- a/src/diffusers/models/adapter.py +++ b/src/diffusers/models/adapter.py @@ -136,7 +136,6 @@ class FullAdapter(nn.Module): downscale_factor: int = 8, ): super().__init__() - print(f"From {self.__class__} channels: {channels}.") in_channels = in_channels * downscale_factor**2 @@ -164,7 +163,7 @@ class FullAdapter(nn.Module): for block in self.body: x = block(x) features.append(x) - print(f"Number of features: {len(features)}") + return features @@ -293,7 +292,7 @@ class LightAdapter(nn.Module): for block in self.body: x = block(x) features.append(x) - print(f"Number of features: {len(features)}") + return features diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index f041bf56ae..5695429489 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -920,7 +920,6 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) is_adapter = mid_block_additional_residual is None and down_block_additional_residuals is not None down_block_res_samples = (sample,) - print(f"From UNet before down blocks: {len(down_block_additional_residuals)}") for downsample_block in self.down_blocks: if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: # For t2i-adapter CrossAttnDownBlock2D @@ -967,7 +966,6 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) encoder_attention_mask=encoder_attention_mask, ) # To support T2I-Adapter-XL - print(f"From UNet in mid block: {len(down_block_additional_residuals)}") if is_adapter and len(down_block_additional_residuals) > 0: sample += down_block_additional_residuals.pop(0) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py index 3f7e19c754..4aa911198a 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -711,7 +711,6 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline): # 7. Denoising loop adapter_state = self.adapter(adapter_input) - print(f"From pipeline (before rejigging): {len(adapter_state)}.") for k, v in enumerate(adapter_state): adapter_state[k] = v * adapter_conditioning_scale if num_images_per_prompt > 1: @@ -720,7 +719,6 @@ class StableDiffusionAdapterPipeline(DiffusionPipeline): if do_classifier_free_guidance: for k, v in enumerate(adapter_state): adapter_state[k] = torch.cat([v] * 2, dim=0) - print(f"From pipeline (after rejigging): {len(adapter_state)}.") num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order with self.progress_bar(total=num_inference_steps) as progress_bar: