diff --git a/src/diffusers/hooks/taylorseer_cache.py b/src/diffusers/hooks/taylorseer_cache.py
index 89d6da3074..3c5d0a2f39 100644
--- a/src/diffusers/hooks/taylorseer_cache.py
+++ b/src/diffusers/hooks/taylorseer_cache.py
@@ -273,7 +273,7 @@ def apply_taylorseer_cache(module: torch.nn.Module, config: TaylorSeerCacheConfi
     logger.debug(f"Special cache identifiers: {special_cache_identifiers}")
 
     for name, submodule in module.named_modules():
-        if skip_compute_identifiers and special_cache_identifiers:
+        if (skip_compute_identifiers and special_cache_identifiers) or (special_cache_identifiers):
             if any(re.fullmatch(identifier, name) for identifier in skip_compute_identifiers) or any(
                 re.fullmatch(identifier, name) for identifier in special_cache_identifiers
             ):