diff --git a/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py b/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py
index e1184964ad..d5f598c311 100644
--- a/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py
+++ b/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py
@@ -1246,8 +1246,7 @@ class UniDiffuserPipeline(DiffusionPipeline):
             )
 
         if reduce_text_emb_dim:
-            with torch.autocast(device_type=device.type):
-                prompt_embeds = self.text_decoder.encode_prefix(prompt_embeds)
+            prompt_embeds = self.text_decoder.encode_prefix(prompt_embeds)
 
         # 4. Encode image, if available; otherwise prepare image latents
         if mode in ["img2text"]: