From f112eab97e50e418c61cf77396c15e661d567914 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Thu, 15 Jan 2026 10:42:42 -1000 Subject: [PATCH] [modular] fix a bug in mellon param & improve docstrings (#12980) * update mellonparams docstring to incude the acutal param definition render in mellon * style --------- Co-authored-by: yiyi@huggingface.co --- .../modular_pipelines/mellon_node_utils.py | 289 ++++++++++++++++-- 1 file changed, 257 insertions(+), 32 deletions(-) diff --git a/src/diffusers/modular_pipelines/mellon_node_utils.py b/src/diffusers/modular_pipelines/mellon_node_utils.py index b4e9463992..f848afe9a3 100644 --- a/src/diffusers/modular_pipelines/mellon_node_utils.py +++ b/src/diffusers/modular_pipelines/mellon_node_utils.py @@ -23,10 +23,18 @@ logger = logging.getLogger(__name__) @dataclass(frozen=True) class MellonParam: """ - Parameter definition for Mellon nodes. + Parameter definition for Mellon nodes. - Use factory methods for common params (e.g., MellonParam.seed()) or create custom ones with MellonParam(name="...", - label="...", type="..."). + Use factory methods for common params (e.g., MellonParam.seed()) or create custom ones with + MellonParam(name="...", label="...", type="..."). + + Example: + ```python + # Custom param + MellonParam(name="my_param", label="My Param", type="float", default=0.5) + # Output in Mellon node definition: + # "my_param": {"label": "My Param", "type": "float", "default": 0.5} + ``` """ name: str @@ -51,14 +59,32 @@ class MellonParam: @classmethod def image(cls) -> "MellonParam": + """ + Image input parameter. + + Mellon node definition: + "image": {"label": "Image", "type": "image", "display": "input"} + """ return cls(name="image", label="Image", type="image", display="input", required_block_params=["image"]) @classmethod def images(cls) -> "MellonParam": + """ + Images output parameter. + + Mellon node definition: + "images": {"label": "Images", "type": "image", "display": "output"} + """ return cls(name="images", label="Images", type="image", display="output", required_block_params=["images"]) @classmethod def control_image(cls, display: str = "input") -> "MellonParam": + """ + Control image parameter for ControlNet. + + Mellon node definition (display="input"): + "control_image": {"label": "Control Image", "type": "image", "display": "input"} + """ return cls( name="control_image", label="Control Image", @@ -69,10 +95,25 @@ class MellonParam: @classmethod def latents(cls, display: str = "input") -> "MellonParam": + """ + Latents parameter. + + Mellon node definition (display="input"): + "latents": {"label": "Latents", "type": "latents", "display": "input"} + + Mellon node definition (display="output"): + "latents": {"label": "Latents", "type": "latents", "display": "output"} + """ return cls(name="latents", label="Latents", type="latents", display=display, required_block_params=["latents"]) @classmethod def image_latents(cls, display: str = "input") -> "MellonParam": + """ + Image latents parameter for img2img workflows. + + Mellon node definition (display="input"): + "image_latents": {"label": "Image Latents", "type": "latents", "display": "input"} + """ return cls( name="image_latents", label="Image Latents", @@ -83,6 +124,12 @@ class MellonParam: @classmethod def first_frame_latents(cls, display: str = "input") -> "MellonParam": + """ + First frame latents for video generation. + + Mellon node definition (display="input"): + "first_frame_latents": {"label": "First Frame Latents", "type": "latents", "display": "input"} + """ return cls( name="first_frame_latents", label="First Frame Latents", @@ -93,6 +140,16 @@ class MellonParam: @classmethod def image_latents_with_strength(cls) -> "MellonParam": + """ + Image latents with strength-based onChange behavior. When connected, shows strength slider; when disconnected, + shows height/width. + + Mellon node definition: + "image_latents": { + "label": "Image Latents", "type": "latents", "display": "input", "onChange": {"false": ["height", + "width"], "true": ["strength"]} + } + """ return cls( name="image_latents", label="Image Latents", @@ -105,16 +162,34 @@ class MellonParam: @classmethod def latents_preview(cls) -> "MellonParam": """ - `Latents Preview` is a special output parameter that is used to preview the latents in the UI. + Latents preview output for visualizing latents in the UI. + + Mellon node definition: + "latents_preview": {"label": "Latents Preview", "type": "latent", "display": "output"} """ return cls(name="latents_preview", label="Latents Preview", type="latent", display="output") @classmethod def embeddings(cls, display: str = "output") -> "MellonParam": + """ + Text embeddings parameter. + + Mellon node definition (display="output"): + "embeddings": {"label": "Text Embeddings", "type": "embeddings", "display": "output"} + + Mellon node definition (display="input"): + "embeddings": {"label": "Text Embeddings", "type": "embeddings", "display": "input"} + """ return cls(name="embeddings", label="Text Embeddings", type="embeddings", display=display) @classmethod def image_embeds(cls, display: str = "output") -> "MellonParam": + """ + Image embeddings parameter for IP-Adapter workflows. + + Mellon node definition (display="output"): + "image_embeds": {"label": "Image Embeddings", "type": "image_embeds", "display": "output"} + """ return cls( name="image_embeds", label="Image Embeddings", @@ -125,6 +200,15 @@ class MellonParam: @classmethod def controlnet_conditioning_scale(cls, default: float = 0.5) -> "MellonParam": + """ + ControlNet conditioning scale slider. + + Mellon node definition (default=0.5): + "controlnet_conditioning_scale": { + "label": "Controlnet Conditioning Scale", "type": "float", "default": 0.5, "min": 0.0, "max": 1.0, + "step": 0.01 + } + """ return cls( name="controlnet_conditioning_scale", label="Controlnet Conditioning Scale", @@ -138,6 +222,15 @@ class MellonParam: @classmethod def control_guidance_start(cls, default: float = 0.0) -> "MellonParam": + """ + Control guidance start timestep. + + Mellon node definition (default=0.0): + "control_guidance_start": { + "label": "Control Guidance Start", "type": "float", "default": 0.0, "min": 0.0, "max": 1.0, "step": + 0.01 + } + """ return cls( name="control_guidance_start", label="Control Guidance Start", @@ -151,6 +244,14 @@ class MellonParam: @classmethod def control_guidance_end(cls, default: float = 1.0) -> "MellonParam": + """ + Control guidance end timestep. + + Mellon node definition (default=1.0): + "control_guidance_end": { + "label": "Control Guidance End", "type": "float", "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01 + } + """ return cls( name="control_guidance_end", label="Control Guidance End", @@ -164,6 +265,12 @@ class MellonParam: @classmethod def prompt(cls, default: str = "") -> "MellonParam": + """ + Text prompt input as textarea. + + Mellon node definition (default=""): + "prompt": {"label": "Prompt", "type": "string", "default": "", "display": "textarea"} + """ return cls( name="prompt", label="Prompt", @@ -175,6 +282,12 @@ class MellonParam: @classmethod def negative_prompt(cls, default: str = "") -> "MellonParam": + """ + Negative prompt input as textarea. + + Mellon node definition (default=""): + "negative_prompt": {"label": "Negative Prompt", "type": "string", "default": "", "display": "textarea"} + """ return cls( name="negative_prompt", label="Negative Prompt", @@ -186,6 +299,12 @@ class MellonParam: @classmethod def strength(cls, default: float = 0.5) -> "MellonParam": + """ + Denoising strength for img2img. + + Mellon node definition (default=0.5): + "strength": {"label": "Strength", "type": "float", "default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01} + """ return cls( name="strength", label="Strength", @@ -199,6 +318,15 @@ class MellonParam: @classmethod def guidance_scale(cls, default: float = 5.0) -> "MellonParam": + """ + CFG guidance scale slider. + + Mellon node definition (default=5.0): + "guidance_scale": { + "label": "Guidance Scale", "type": "float", "display": "slider", "default": 5.0, "min": 1.0, "max": + 30.0, "step": 0.1 + } + """ return cls( name="guidance_scale", label="Guidance Scale", @@ -212,6 +340,12 @@ class MellonParam: @classmethod def height(cls, default: int = 1024) -> "MellonParam": + """ + Image height in pixels. + + Mellon node definition (default=1024): + "height": {"label": "Height", "type": "int", "default": 1024, "min": 64, "step": 8} + """ return cls( name="height", label="Height", @@ -224,12 +358,26 @@ class MellonParam: @classmethod def width(cls, default: int = 1024) -> "MellonParam": + """ + Image width in pixels. + + Mellon node definition (default=1024): + "width": {"label": "Width", "type": "int", "default": 1024, "min": 64, "step": 8} + """ return cls( name="width", label="Width", type="int", default=default, min=64, step=8, required_block_params=["width"] ) @classmethod def seed(cls, default: int = 0) -> "MellonParam": + """ + Random seed with randomize button. + + Mellon node definition (default=0): + "seed": { + "label": "Seed", "type": "int", "default": 0, "min": 0, "max": 4294967295, "display": "random" + } + """ return cls( name="seed", label="Seed", @@ -243,6 +391,14 @@ class MellonParam: @classmethod def num_inference_steps(cls, default: int = 25) -> "MellonParam": + """ + Number of denoising steps slider. + + Mellon node definition (default=25): + "num_inference_steps": { + "label": "Steps", "type": "int", "default": 25, "min": 1, "max": 100, "display": "slider" + } + """ return cls( name="num_inference_steps", label="Steps", @@ -256,6 +412,12 @@ class MellonParam: @classmethod def num_frames(cls, default: int = 81) -> "MellonParam": + """ + Number of video frames slider. + + Mellon node definition (default=81): + "num_frames": {"label": "Frames", "type": "int", "default": 81, "min": 1, "max": 480, "display": "slider"} + """ return cls( name="num_frames", label="Frames", @@ -269,6 +431,12 @@ class MellonParam: @classmethod def layers(cls, default: int = 4) -> "MellonParam": + """ + Number of layers slider (for layered diffusion). + + Mellon node definition (default=4): + "layers": {"label": "Layers", "type": "int", "default": 4, "min": 1, "max": 10, "display": "slider"} + """ return cls( name="layers", label="Layers", @@ -282,15 +450,24 @@ class MellonParam: @classmethod def videos(cls) -> "MellonParam": + """ + Video output parameter. + + Mellon node definition: + "videos": {"label": "Videos", "type": "video", "display": "output"} + """ return cls(name="videos", label="Videos", type="video", display="output", required_block_params=["videos"]) @classmethod def vae(cls) -> "MellonParam": """ - VAE model info dict. + VAE model input. - Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve - the actual model. + Mellon node definition: + "vae": {"label": "VAE", "type": "diffusers_auto_model", "display": "input"} + + Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use + components.get_one(model_id) to retrieve the actual model. """ return cls( name="vae", label="VAE", type="diffusers_auto_model", display="input", required_block_params=["vae"] @@ -299,10 +476,13 @@ class MellonParam: @classmethod def image_encoder(cls) -> "MellonParam": """ - Image Encoder model info dict. + Image encoder model input. - Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve - the actual model. + Mellon node definition: + "image_encoder": {"label": "Image Encoder", "type": "diffusers_auto_model", "display": "input"} + + Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use + components.get_one(model_id) to retrieve the actual model. """ return cls( name="image_encoder", @@ -315,30 +495,39 @@ class MellonParam: @classmethod def unet(cls) -> "MellonParam": """ - Denoising model (UNet/Transformer) info dict. + Denoising model (UNet/Transformer) input. - Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve - the actual model. + Mellon node definition: + "unet": {"label": "Denoise Model", "type": "diffusers_auto_model", "display": "input"} + + Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use + components.get_one(model_id) to retrieve the actual model. """ return cls(name="unet", label="Denoise Model", type="diffusers_auto_model", display="input") @classmethod def scheduler(cls) -> "MellonParam": """ - Scheduler model info dict. + Scheduler model input. - Contains keys like 'model_id', 'repo_id' etc. Use components.get_one(model_id) to retrieve the actual - scheduler. + Mellon node definition: + "scheduler": {"label": "Scheduler", "type": "diffusers_auto_model", "display": "input"} + + Note: The value received is a model info dict with keys like 'model_id', 'repo_id'. Use + components.get_one(model_id) to retrieve the actual scheduler. """ return cls(name="scheduler", label="Scheduler", type="diffusers_auto_model", display="input") @classmethod def controlnet(cls) -> "MellonParam": """ - ControlNet model info dict. + ControlNet model input. - Contains keys like 'model_id', 'repo_id', 'execution_device' etc. Use components.get_one(model_id) to retrieve - the actual model. + Mellon node definition: + "controlnet": {"label": "ControlNet Model", "type": "diffusers_auto_model", "display": "input"} + + Note: The value received is a model info dict with keys like 'model_id', 'repo_id', 'execution_device'. Use + components.get_one(model_id) to retrieve the actual model. """ return cls( name="controlnet", @@ -351,12 +540,17 @@ class MellonParam: @classmethod def text_encoders(cls) -> "MellonParam": """ - Dict of text encoder model info dicts. + Text encoders dict input (multiple encoders). - Structure: { - 'text_encoder': {'model_id': ..., 'execution_device': ..., ...}, 'tokenizer': {'model_id': ..., ...}, - 'repo_id': '...' - } Use components.get_one(model_id) to retrieve each model. + Mellon node definition: + "text_encoders": {"label": "Text Encoders", "type": "diffusers_auto_models", "display": "input"} + + Note: The value received is a dict of model info dicts: + { + 'text_encoder': {'model_id': ..., 'execution_device': ..., ...}, 'tokenizer': {'model_id': ..., ...}, + 'repo_id': '...' + } + Use components.get_one(model_id) to retrieve each model. """ return cls( name="text_encoders", @@ -369,15 +563,20 @@ class MellonParam: @classmethod def controlnet_bundle(cls, display: str = "input") -> "MellonParam": """ - ControlNet bundle containing model info and processed control inputs. + ControlNet bundle containing model and processed control inputs. Output from ControlNet node, input to Denoise + node. - Structure: { - 'controlnet': {'model_id': ..., ...}, # controlnet model info dict 'control_image': ..., # processed - control image/embeddings 'controlnet_conditioning_scale': ..., ... # other inputs expected by denoise - blocks - } + Mellon node definition (display="input"): + "controlnet_bundle": {"label": "ControlNet", "type": "custom_controlnet", "display": "input"} - Output from Controlnet node, input to Denoise node. + Mellon node definition (display="output"): + "controlnet_bundle": {"label": "ControlNet", "type": "custom_controlnet", "display": "output"} + + Note: The value is a dict containing: + { + 'controlnet': {'model_id': ..., ...}, # controlnet model info 'control_image': ..., # processed control + image/embeddings 'controlnet_conditioning_scale': ..., # and other denoise block inputs + } """ return cls( name="controlnet_bundle", @@ -389,10 +588,25 @@ class MellonParam: @classmethod def ip_adapter(cls) -> "MellonParam": + """ + IP-Adapter input. + + Mellon node definition: + "ip_adapter": {"label": "IP Adapter", "type": "custom_ip_adapter", "display": "input"} + """ return cls(name="ip_adapter", label="IP Adapter", type="custom_ip_adapter", display="input") @classmethod def guider(cls) -> "MellonParam": + """ + Custom guider input. When connected, hides the guidance_scale slider. + + Mellon node definition: + "guider": { + "label": "Guider", "type": "custom_guider", "display": "input", "onChange": {false: ["guidance_scale"], + true: []} + } + """ return cls( name="guider", label="Guider", @@ -403,6 +617,12 @@ class MellonParam: @classmethod def doc(cls) -> "MellonParam": + """ + Documentation output for inspecting the underlying modular pipeline. + + Mellon node definition: + "doc": {"label": "Doc", "type": "string", "display": "output"} + """ return cls(name="doc", label="Doc", type="string", display="output") @@ -415,6 +635,7 @@ DEFAULT_NODE_SPECS = { MellonParam.height(), MellonParam.seed(), MellonParam.num_inference_steps(), + MellonParam.num_frames(), MellonParam.guidance_scale(), MellonParam.strength(), MellonParam.image_latents_with_strength(), @@ -669,6 +890,9 @@ class MellonPipelineConfig: @property def node_params(self) -> Dict[str, Any]: """Lazily compute node_params from node_specs.""" + if self.node_specs is None: + return self._node_params + params = {} for node_type, spec in self.node_specs.items(): if spec is None: @@ -711,7 +935,8 @@ class MellonPipelineConfig: Note: The mellon_params are already in Mellon format when loading from JSON. """ instance = cls.__new__(cls) - instance.node_params = data.get("node_params", {}) + instance.node_specs = None + instance._node_params = data.get("node_params", {}) instance.label = data.get("label", "") instance.default_repo = data.get("default_repo", "") instance.default_dtype = data.get("default_dtype", "")