diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py
index ebe515f1b3..71e49ae5a8 100644
--- a/src/diffusers/pipelines/pipeline_utils.py
+++ b/src/diffusers/pipelines/pipeline_utils.py
@@ -531,8 +531,7 @@ class DiffusionPipeline(ConfigMixin):
             )
         else:
             cached_folder = pretrained_model_name_or_path
-
-        config_dict = cls.load_config(cached_folder)
+            config_dict = cls.load_config(cached_folder)
 
         # 2. Load the pipeline class, if using custom module then load it from the hub
         # if we load from explicit class, let's use it