diff --git a/src/diffusers/loaders.py b/src/diffusers/loaders.py
index bc40cf9a18..f63532b84e 100644
--- a/src/diffusers/loaders.py
+++ b/src/diffusers/loaders.py
@@ -1922,7 +1922,7 @@ class LoraLoaderMixin:
                 diffusers_name = diffusers_name.replace("emb.layers", "time_emb_proj")
 
                 # SDXL specificity.
-                if "emb" in diffusers_name and "time" not in diffusers_name:
+                if "emb" in diffusers_name and "time.emb.proj" not in diffusers_name:
                     pattern = r"\.\d+(?=\D*$)"
                     diffusers_name = re.sub(pattern, "", diffusers_name, count=1)
                 if ".in." in diffusers_name:
@@ -1935,7 +1935,7 @@ class LoraLoaderMixin:
                     diffusers_name = diffusers_name.replace("skip.connection", "conv_shortcut")
 
                 # LyCORIS specificity.
-                if "time" in diffusers_name:
+                if "time.emb.proj" in diffusers_name:
                     diffusers_name = diffusers_name.replace("time.emb.proj", "time_emb_proj")
                 if "conv.shortcut" in diffusers_name:
                     diffusers_name = diffusers_name.replace("conv.shortcut", "conv_shortcut")