diff --git a/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py b/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py
index d5f598c311..e1184964ad 100644
--- a/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py
+++ b/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py
@@ -1246,7 +1246,8 @@ class UniDiffuserPipeline(DiffusionPipeline):
             )
 
         if reduce_text_emb_dim:
-            prompt_embeds = self.text_decoder.encode_prefix(prompt_embeds)
+            with torch.autocast(device_type=device.type):
+                prompt_embeds = self.text_decoder.encode_prefix(prompt_embeds)
 
         # 4. Encode image, if available; otherwise prepare image latents
         if mode in ["img2text"]: