diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index 23c4b29a53..50fbbedc99 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -495,6 +495,8 @@ class DiffusionPipeline(ConfigMixin):
                 )
                 if is_safetensors_compatible(info):
                     ignore_patterns.append("*.bin")
+                else:
+                    ignore_patterns.append("*.safetensors")
 
             # download all allow_patterns
             cached_folder = snapshot_download(