load_components by default only load components that are not already loaded

src/diffusers/modular_pipelines/modular_pipeline.py

@@ -2142,6 +2142,7 @@ class ModularPipeline(ConfigMixin, PushToHubMixin):
                 name
                 for name in self._component_specs.keys()
                 if self._component_specs[name].default_creation_method == "from_pretrained"
+                and getattr(self, name, None) is None
             ]
         elif isinstance(names, str):
             names = [names]