AI/diffusers
1
0
Fork 0
mirror of https://github.com/huggingface/diffusers.git synced 2026-01-27 17:22:53 +03:00
Code Activity

[modular] fix a bug in mellon param & improve docstrings (#12980)

Browse Source 
* update mellonparams docstring to incude the acutal param definition render in mellon

* style

---------

Co-authored-by: yiyi@huggingface.co <yiyi@ip-26-0-160-103.ec2.internal>
This commit is contained in:
YiYi Xu
2026-01-15 10:42:42 -10:00
committed by GitHub
parent 61f175660a
commit f112eab97e
1 changed files with 257 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

289
src/diffusers/modular_pipelines/mellon_node_utils.py
View File

@@ -23,10 +23,18 @@ logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class MellonParam:
"""
Parameter definition for Mellon nodes.
Parameter definition for Mellon nodes.
Use factory methods for common params (e.g., MellonParam.seed()) or create custom ones with MellonParam(name="...",
label="...", type="...").
Use factory methods for common params (e.g., MellonParam.seed()) or create custom ones with
MellonParam(name="...", label="...", type="...").
Example:
```python
# Custom param
MellonParam(name="my_param", label="My Param", type="float", default=0.5)
# Output in Mellon node definition:
# "my_param": {"label": "My Param", "type": "float", "default": 0.5}
```
"""
name: str
@@ -51,14 +59,32 @@ class MellonParam:
@classmethod
def image(cls) -> "MellonParam":
"""
Image input parameter.
Mellon node definition:
"image": {"label": "Image", "type": "image", "display": "input"}
"""
return cls(name="image", label="Image", type="image", display="input", required_block_params=["image"])
@classmethod
def images(cls) -> "MellonParam":
"""
Images output parameter.
Mellon node definition:
"images": {"label": "Images", "type": "image", "display": "output"}
"""
return cls(name="images", label="Images", type="image", display="output", required_block_params=["images"])
@classmethod
def control_image(cls, display: str = "input") -> "MellonParam":
"""
Control image parameter for ControlNet.
Mellon node definition (display="input"):
"control_image": {"label": "Control Image", "type": "image", "display": "input"}
"""
return cls(
name="control_image",
label="Control Image",
@@ -69,10 +95,25 @@ class MellonParam:
@classmethod
def latents(cls, display: str = "input") -> "MellonParam":
"""
Latents parameter.
Mellon node definition (display="input"):
"latents": {"label": "Latents", "type": "latents", "display": "input"}
Mellon node definition (display="output"):
"latents": {"label": "Latents", "type": "latents", "display": "output"}
"""
return cls(name="latents", label="Latents", type="latents", display=display, required_block_params=["latents"])
@classmethod
def image_latents(cls, display: str = "input") -> "MellonParam":
"""
Image latents parameter for img2img workflows.
Mellon node definition (display="input"):
"image_latents": {"label": "Image Latents", "type": "latents", "display": "input"}
"""
return cls(
name="image_latents",
label="Image Latents",
@@ -83,6 +124,12 @@ class MellonParam:
@classmethod
def first_frame_latents(cls, display: str = "input") -> "MellonParam":
"""
First frame latents for video generation.
Mellon node definition (display="input"):
"first_frame_latents": {"label": "First Frame Latents", "type": "latents", "display": "input"}
"""
return cls(
name="first_frame_latents",
label="First Frame Latents",
@@ -93,6 +140,16 @@ class MellonParam:
@classmethod
def image_latents_with_strength(cls) -> "MellonParam":
"""
Image latents with strength-based onChange behavior. When connected, shows strength slider; when disconnected,
shows height/width.
Mellon node definition:
"image_latents": {
"label": "Image Latents", "type": "latents", "display": "input", "onChange": {"false": ["height",
"width"], "true": ["strength"]}
}
"""
return cls(
name="image_latents",
label="Image Latents",
@@ -105,16 +162,34 @@ class MellonParam:
@classmethod
def latents_preview(cls) -> "MellonParam":
"""
`Latents Preview` is a special output parameter that is used to preview the latents in the UI.
Latents preview output for visualizing latents in the UI.
Mellon node definition:
"latents_preview": {"label": "Latents Preview", "type": "latent", "display": "output"}
"""
return cls(name="latents_preview", label="Latents Preview", type="latent", display="output")
@classmethod
def embeddings(cls, display: str = "output") -> "MellonParam":
"""
Text embeddings parameter.
Mellon node definition (display="output"):
"embeddings": {"label": "Text Embeddings", "type": "embeddings", "display": "output"}
Mellon node definition (display="input"):
"embeddings": {"label": "Text Embeddings", "type": "embeddings", "display": "input"}
"""
return cls(name="embeddings", label="Text Embeddings", type="embeddings", display=display)
@classmethod
def image_embeds(cls, display: str = "output") -> "MellonParam":
"""
Image embeddings parameter for IP-Adapter workflows.
Mellon node definition (display="output"):
"image_embeds": {"label": "Image Embeddings", "type": "image_embeds", "display": "output"}
"""
return cls(
name="image_embeds",
label="Image Embeddings",
@@ -125,6 +200,15 @@ class MellonParam:
@classmethod
def controlnet_conditioning_scale(cls, default: float = 0.5) -> "MellonParam":
"""
ControlNet conditioning scale slider.
Mellon node definition (default=0.5):
"controlnet_conditioning_scale": {
"label": "Controlnet Conditioning Scale", "type": "float", "default": 0.5, "min": 0.0, "max": 1.0,
"step": 0.01
}
"""
return cls(
name="controlnet_conditioning_scale",
label="Controlnet Conditioning Scale",
@@ -138,6 +222,15 @@ class MellonParam:
@classmethod
def control_guidance_start(cls, default: float = 0.0) -> "MellonParam":
"""
Control guidance start timestep.
Mellon node definition (default=0.0):
"control_guidance_start": {
"label": "Control Guidance Start", "type": "float", "default": 0.0, "min": 0.0, "max": 1.0, "step":
0.01
}
"""
return cls(
name="control_guidance_start",
label="Control Guidance Start",
@@ -151,6 +244,14 @@ class MellonParam:
@classmethod
def control_guidance_end(cls, default: float = 1.0) -> "MellonParam":
"""
Control guidance end timestep.
Mellon node definition (default=1.0):
"control_guidance_end": {
"label": "Control Guidance End", "type": "float", "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01
}
"""
return cls(
name="control_guidance_end",
label="Control Guidance End",
@@ -164,6 +265,12 @@ class MellonParam:
@classmethod
def prompt(cls, default: str = "") -> "MellonParam":
"""
Text prompt input as textarea.
Mellon node definition (default=""):
"prompt": {"label": "Prompt", "type": "string", "default": "", "display": "textarea"}
"""
return cls(
name="prompt",
label="Prompt",
@@ -175,6 +282,12 @@ class MellonParam:
@classmethod
def negative_prompt(cls, default: str = "") -> "MellonParam":
"""
Negative prompt input as textarea.
Mellon node definition (default=""):
"negative_prompt": {"label": "Negative Prompt", "type": "string", "default": "", "display": "textarea"}
"""
return cls(
name="negative_prompt",
label="Negative Prompt",
@@ -186,6 +299,12 @@ class MellonParam:
@classmethod
def strength(cls, default: float = 0.5) -> "MellonParam":
"""
Denoising strength for img2img.
Mellon node definition (default=0.5):
"strength": {"label": "Strength", "type": "float", "default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}
"""
return cls(
name="strength",
label="Strength",
@@ -199,6 +318,15 @@ class MellonParam:
@classmethod
def guidance_scale(cls, default: float = 5.0) -> "MellonParam":
"""
CFG guidance scale slider.
Mellon node definition (default=5.0):
"guidance_scale": {
"label": "Guidance Scale", "type": "float", "display": "slider", "default": 5.0, "min": 1.0, "max":
30.0, "step": 0.1
}
"""
return cls(
name="guidance_scale",
label="Guidance Scale",
@@ -212,6 +340,12 @@ class MellonParam:
@classmethod
def height(cls, default: int = 1024) -> "MellonParam":
"""
Image height in pixels.
Mellon node definition (default=1024):
"height": {"label": "Height", "type": "int", "default": 1024, "min": 64, "step": 8}
"""
return cls(
name="height",
label="Height",
@@ -224,12 +358,26 @@ class MellonParam:
@classmethod
def width(cls, default: int = 1024) -> "MellonParam":
"""
Image width in pixels.
Mellon node definition (default=1024):
"width": {"label": "Width", "type": "int", "default": 1024, "min": 64, "step": 8}
"""
return cls(
name="width", label="Width", type="int", default=default, min=64, step=8, required_block_params=["width"]
)
@classmethod
def seed(cls, default: int = 0) -> "MellonParam":
"""
Random seed with randomize button.
Mellon node definition (default=0):
"seed": {
"label": "Seed", "type": "int", "default": 0, "min": 0, "max": 4294967295, "display": "random"
}
"""
return cls(
name="seed",
label="Seed",
@@ -243,6 +391,14 @@ class MellonParam:
@classmethod
def num_inference_steps(cls, default: int = 25) -> "MellonParam":
"""
Number of denoising steps slider.
Mellon node definition (default=25):
"num_inference_steps": {
"label": "Steps", "type": "int", "default": 25, "min": 1, "max": 100, "display": "slider"
}
"""
return cls(
name="num_inference_steps",
label="Steps",
@@ -256,6 +412,12 @@ class MellonParam:
@classmethod
def num_frames(cls, default: int = 81) -> "MellonParam":
"""
Number of video frames slider.
Mellon node definition (default=81):
"num_frames": {"label": "Frames", "type": "int", "default": 81, "min": 1, "max": 480, "display": "slider"}
"""
return cls(
name="num_frames",
label="Frames",
@@ -269,6 +431,12 @@ class MellonParam:
@classmethod
def layers(cls, default: int = 4) -> "MellonParam":
"""
Number of layers slider (for layered diffusion).
Mellon node definition (default=4):
"layers": {"label": "Layers", "type": "int", "default": 4, "min": 1, "max": 10, "display": "slider"}
"""
return cls(
name="layers",
label="Layers",
@@ -282,15 +450,24 @@ class MellonParam:
@classmethod
def videos(cls) -> "MellonParam":
"""
Video output parameter.
Mellon node definition:
"videos": {"label": "Videos", "type": "video", "display": "output"}
"""
return cls(name="videos", label="Videos", type="video", display="output", required_block_params=["videos"])
@classmethod
def vae(cls) -> "MellonParam":
"""
VAE model info dict.
VAE model input.
Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve
the actual model.
Mellon node definition:
"vae": {"label": "VAE", "type": "diffusers_auto_model", "display": "input"}
Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use
components.get_one(model_id) to retrieve the actual model.
"""
return cls(
name="vae", label="VAE", type="diffusers_auto_model", display="input", required_block_params=["vae"]
@@ -299,10 +476,13 @@ class MellonParam:
@classmethod
def image_encoder(cls) -> "MellonParam":
"""
Image Encoder model info dict.
Image encoder model input.
Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve
the actual model.
Mellon node definition:
"image_encoder": {"label": "Image Encoder", "type": "diffusers_auto_model", "display": "input"}
Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use
components.get_one(model_id) to retrieve the actual model.
"""
return cls(
name="image_encoder",
@@ -315,30 +495,39 @@ class MellonParam:
@classmethod
def unet(cls) -> "MellonParam":
"""
Denoising model (UNet/Transformer) info dict.
Denoising model (UNet/Transformer) input.
Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve
the actual model.
Mellon node definition:
"unet": {"label": "Denoise Model", "type": "diffusers_auto_model", "display": "input"}
Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use
components.get_one(model_id) to retrieve the actual model.
"""
return cls(name="unet", label="Denoise Model", type="diffusers_auto_model", display="input")
@classmethod
def scheduler(cls) -> "MellonParam":
"""
Scheduler model info dict.
Scheduler model input.
Contains keys like 'model_id', 'repo_id' etc. Use components.get_one(model_id) to retrieve the actual
scheduler.
Mellon node definition:
"scheduler": {"label": "Scheduler", "type": "diffusers_auto_model", "display": "input"}
Note: The value received is a model info dict with keys like 'model_id', 'repo_id'. Use
components.get_one(model_id) to retrieve the actual scheduler.
"""
return cls(name="scheduler", label="Scheduler", type="diffusers_auto_model", display="input")
@classmethod
def controlnet(cls) -> "MellonParam":
"""
ControlNet model info dict.
ControlNet model input.
Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve
the actual model.
Mellon node definition:
"controlnet": {"label": "ControlNet Model", "type": "diffusers_auto_model", "display": "input"}
Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use
components.get_one(model_id) to retrieve the actual model.
"""
return cls(
name="controlnet",
@@ -351,12 +540,17 @@ class MellonParam:
@classmethod
def text_encoders(cls) -> "MellonParam":
"""
Dict of text encoder model info dicts.
Text encoders dict input (multiple encoders).
Structure: {
'text_encoder': {'model_id': ..., 'execution_device': ..., ...}, 'tokenizer': {'model_id': ..., ...},
'repo_id': '...'
} Use components.get_one(model_id) to retrieve each model.
Mellon node definition:
"text_encoders": {"label": "Text Encoders", "type": "diffusers_auto_models", "display": "input"}
Note: The value received is a dict of model info dicts:
{
'text_encoder': {'model_id': ..., 'execution_device': ..., ...}, 'tokenizer': {'model_id': ..., ...},
'repo_id': '...'
}
Use components.get_one(model_id) to retrieve each model.
"""
return cls(
name="text_encoders",
@@ -369,15 +563,20 @@ class MellonParam:
@classmethod
def controlnet_bundle(cls, display: str = "input") -> "MellonParam":
"""
ControlNet bundle containing model info and processed control inputs.
ControlNet bundle containing model and processed control inputs. Output from ControlNet node, input to Denoise
node.
Structure: {
'controlnet': {'model_id': ..., ...}, # controlnet model info dict 'control_image': ..., # processed
control image/embeddings 'controlnet_conditioning_scale': ..., ... # other inputs expected by denoise
blocks
}
Mellon node definition (display="input"):
"controlnet_bundle": {"label": "ControlNet", "type": "custom_controlnet", "display": "input"}
Output from Controlnet node, input to Denoise node.
Mellon node definition (display="output"):
"controlnet_bundle": {"label": "ControlNet", "type": "custom_controlnet", "display": "output"}
Note: The value is a dict containing:
{
'controlnet': {'model_id': ..., ...}, # controlnet model info 'control_image': ..., # processed control
image/embeddings 'controlnet_conditioning_scale': ..., # and other denoise block inputs
}
"""
return cls(
name="controlnet_bundle",
@@ -389,10 +588,25 @@ class MellonParam:
@classmethod
def ip_adapter(cls) -> "MellonParam":
"""
IP-Adapter input.
Mellon node definition:
"ip_adapter": {"label": "IP Adapter", "type": "custom_ip_adapter", "display": "input"}
"""
return cls(name="ip_adapter", label="IP Adapter", type="custom_ip_adapter", display="input")
@classmethod
def guider(cls) -> "MellonParam":
"""
Custom guider input. When connected, hides the guidance_scale slider.
Mellon node definition:
"guider": {
"label": "Guider", "type": "custom_guider", "display": "input", "onChange": {false: ["guidance_scale"],
true: []}
}
"""
return cls(
name="guider",
label="Guider",
@@ -403,6 +617,12 @@ class MellonParam:
@classmethod
def doc(cls) -> "MellonParam":
"""
Documentation output for inspecting the underlying modular pipeline.
Mellon node definition:
"doc": {"label": "Doc", "type": "string", "display": "output"}
"""
return cls(name="doc", label="Doc", type="string", display="output")
@@ -415,6 +635,7 @@ DEFAULT_NODE_SPECS = {
MellonParam.height(),
MellonParam.seed(),
MellonParam.num_inference_steps(),
MellonParam.num_frames(),
MellonParam.guidance_scale(),
MellonParam.strength(),
MellonParam.image_latents_with_strength(),
@@ -669,6 +890,9 @@ class MellonPipelineConfig:
@property
def node_params(self) -> Dict[str, Any]:
"""Lazily compute node_params from node_specs."""
if self.node_specs is None:
return self._node_params
params = {}
for node_type, spec in self.node_specs.items():
if spec is None:
@@ -711,7 +935,8 @@ class MellonPipelineConfig:
Note: The mellon_params are already in Mellon format when loading from JSON.
"""
instance = cls.__new__(cls)
instance.node_params = data.get("node_params", {})
instance.node_specs = None
instance._node_params = data.get("node_params", {})
instance.label = data.get("label", "")
instance.default_repo = data.get("default_repo", "")
instance.default_dtype = data.get("default_dtype", "")