From 6398fbc391519ce033d97d23925fd13ec4ed85a7 Mon Sep 17 00:00:00 2001 From: Hengyue-Bi <616730841@qq.com> Date: Tue, 15 Jul 2025 02:54:38 +0800 Subject: [PATCH] Fix: Align VAE processing in ControlNet SD3 training with inference (#11909) Fix: Apply vae_shift_factor in ControlNet SD3 training --- examples/controlnet/train_controlnet_sd3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/controlnet/train_controlnet_sd3.py b/examples/controlnet/train_controlnet_sd3.py index 746063f9d6..592e5d7766 100644 --- a/examples/controlnet/train_controlnet_sd3.py +++ b/examples/controlnet/train_controlnet_sd3.py @@ -1330,7 +1330,7 @@ def main(args): # controlnet(s) inference controlnet_image = batch["conditioning_pixel_values"].to(dtype=weight_dtype) controlnet_image = vae.encode(controlnet_image).latent_dist.sample() - controlnet_image = controlnet_image * vae.config.scaling_factor + controlnet_image = (controlnet_image - vae.config.shift_factor) * vae.config.scaling_factor control_block_res_samples = controlnet( hidden_states=noisy_model_input,