diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py
index 93a6e15449..5fef0f361e 100644
--- a/src/diffusers/loaders/single_file.py
+++ b/src/diffusers/loaders/single_file.py
@@ -219,7 +219,7 @@ def build_additional_components(
         )
         return paint_by_example_components
 
-    if pipeline_class_name == "StableDiffusionXLImg2ImgPipeline":
+    if pipeline_class_name in ["StableDiffusionXLImg2ImgPipeline", "StableDiffusionXLInpaintPipeline"]:
         model_type = infer_model_type(pipeline_class_name, original_config)
         is_refiner = model_type == "SDXL-Refiner"
         return {