From 371f7659086ecedcc3331b62bf11b671c1502927 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Mon, 12 Feb 2024 17:30:22 +0530 Subject: [PATCH] [Diffusers -> Original SD conversion] fix things (#6933) * fix: bias loading bug * fixes for SDXL * apply changes to the conversion script to match single_file_utils.py * do transpose to match the single file loading logic. --- scripts/convert_diffusers_to_original_sdxl.py | 12 +++++++++++- ...convert_diffusers_to_original_stable_diffusion.py | 5 ++++- src/diffusers/loaders/single_file_utils.py | 1 - 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/convert_diffusers_to_original_sdxl.py b/scripts/convert_diffusers_to_original_sdxl.py index 1f11ef4570..648d0376f7 100644 --- a/scripts/convert_diffusers_to_original_sdxl.py +++ b/scripts/convert_diffusers_to_original_sdxl.py @@ -167,7 +167,10 @@ vae_conversion_map_attn = [ def reshape_weight_for_sd(w): # convert HF linear weights to SD conv2d weights - return w.reshape(*w.shape, 1, 1) + if not w.ndim == 1: + return w.reshape(*w.shape, 1, 1) + else: + return w def convert_vae_state_dict(vae_state_dict): @@ -321,11 +324,18 @@ if __name__ == "__main__": vae_state_dict = convert_vae_state_dict(vae_state_dict) vae_state_dict = {"first_stage_model." + k: v for k, v in vae_state_dict.items()} + # Convert text encoder 1 text_enc_dict = convert_openai_text_enc_state_dict(text_enc_dict) text_enc_dict = {"conditioner.embedders.0.transformer." + k: v for k, v in text_enc_dict.items()} + # Convert text encoder 2 text_enc_2_dict = convert_openclip_text_enc_state_dict(text_enc_2_dict) text_enc_2_dict = {"conditioner.embedders.1.model." + k: v for k, v in text_enc_2_dict.items()} + # We call the `.T.contiguous()` to match what's done in + # https://github.com/huggingface/diffusers/blob/84905ca7287876b925b6bf8e9bb92fec21c78764/src/diffusers/loaders/single_file_utils.py#L1085 + text_enc_2_dict["conditioner.embedders.1.model.text_projection"] = text_enc_2_dict.pop( + "conditioner.embedders.1.model.text_projection.weight" + ).T.contiguous() # Put together new checkpoint state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict, **text_enc_2_dict} diff --git a/scripts/convert_diffusers_to_original_stable_diffusion.py b/scripts/convert_diffusers_to_original_stable_diffusion.py index cc90a51317..d1b7df070c 100644 --- a/scripts/convert_diffusers_to_original_stable_diffusion.py +++ b/scripts/convert_diffusers_to_original_stable_diffusion.py @@ -170,7 +170,10 @@ vae_extra_conversion_map = [ def reshape_weight_for_sd(w): # convert HF linear weights to SD conv2d weights - return w.reshape(*w.shape, 1, 1) + if not w.ndim == 1: + return w.reshape(*w.shape, 1, 1) + else: + return w def convert_vae_state_dict(vae_state_dict): diff --git a/src/diffusers/loaders/single_file_utils.py b/src/diffusers/loaders/single_file_utils.py index 1df964fe41..3a9f6e8823 100644 --- a/src/diffusers/loaders/single_file_utils.py +++ b/src/diffusers/loaders/single_file_utils.py @@ -1112,7 +1112,6 @@ def create_text_encoder_from_open_clip_checkpoint( text_model_dict[diffusers_key + ".q_proj.bias"] = weight_value[:text_proj_dim] text_model_dict[diffusers_key + ".k_proj.bias"] = weight_value[text_proj_dim : text_proj_dim * 2] text_model_dict[diffusers_key + ".v_proj.bias"] = weight_value[text_proj_dim * 2 :] - else: text_model_dict[diffusers_key] = checkpoint[key]