diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py
index a35b1fb1ec..da0a5e9c13 100644
--- a/src/diffusers/loaders/single_file.py
+++ b/src/diffusers/loaders/single_file.py
@@ -163,12 +163,6 @@ def build_component(
         )
         return controlnet_components
 
-    if component_name == "adapter":
-        adapter_components = create_adapter_model(
-            pipeline_class_name, original_config, checkpoint, checkpoint_path_or_dict, **kwargs
-        )
-        return adapter_components
-
     if component_name == "scheduler":
         scheduler_components = create_scheduler(
             pipeline_class_name, original_config, checkpoint, checkpoint_path_or_dict, **kwargs