Avoid duplicating PyTorch + safetensors downloads. (#1836)

src/diffusers/pipeline_utils.py

@@ -495,6 +495,8 @@ class DiffusionPipeline(ConfigMixin):
                 )
                 if is_safetensors_compatible(info):
                     ignore_patterns.append("*.bin")
+                else:
+                    ignore_patterns.append("*.safetensors")
 
             # download all allow_patterns
             cached_folder = snapshot_download(