mirror of
https://github.com/huggingface/diffusers.git
synced 2026-10-24 20:59:27 +03:00
[AnyFlow] FAR: move chunk_partition default into transformer config
- AnyFlowFARTransformer3DModel.__init__ now accepts chunk_partition via @register_to_config (default (1, 3, 3, 3, 3, 3, 3, 2) for the released 81-frame checkpoints, matching the field on Hub). - AnyFlowFARPipeline.__call__ no longer requires chunk_partition; defaults to self.transformer.config.chunk_partition. Per-call override still supported for V2V / non-default num_frames. - Drop the AnyFlowFARPipeline.default_chunk_partition class attribute. - Update docs (en pipelines/models, zh using-diffusers) and the conversion script to match.
This commit is contained in:
@@ -25,8 +25,10 @@ the v0.35.1 Wan2.1 backbone with three additions:
|
||||
[`AnyFlowTransformer3DModel`](anyflow_transformer3d)) — every forward call conditions on both the source
|
||||
timestep ``t`` and the target timestep ``r``.
|
||||
|
||||
The chunk schedule (`chunk_partition`) is **not** baked into the model config. It is a per-call argument to
|
||||
`forward`, so the same checkpoint handles different `num_frames` configurations without retraining.
|
||||
The default chunk schedule (`chunk_partition`) is stored in the model config; the released NVIDIA AnyFlow-FAR
|
||||
checkpoints use `[1, 3, 3, 3, 3, 3, 3, 2]` for the canonical 81-frame setting. `forward` accepts a per-call
|
||||
`chunk_partition` override, so the same checkpoint also handles other `num_frames` configurations without
|
||||
retraining.
|
||||
|
||||
```python
|
||||
from diffusers import AnyFlowFARTransformer3DModel
|
||||
|
||||
@@ -70,10 +70,10 @@ clip for V2V continuation. If you already have pre-encoded latents in the model
|
||||
``video_latents=<tensor>`` to skip VAE encoding. ``video`` and ``video_latents`` are mutually exclusive.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> `AnyFlowFARPipeline.default_chunk_partition = [1, 3, 3, 3, 3, 3, 3, 2]` (sum 21) is matched to the
|
||||
> released checkpoints' canonical 81 raw frames (21 latent frames at the VAE temporal stride of 4). When
|
||||
> you change `num_frames`, you must also pass a matching `chunk_partition` summing to
|
||||
> `(num_frames - 1) // 4 + 1`, otherwise the pipeline raises an `AssertionError`.
|
||||
> The released checkpoints bake `chunk_partition=[1, 3, 3, 3, 3, 3, 3, 2]` (sum 21) into the transformer
|
||||
> config, matched to the canonical 81 raw frames (21 latent frames at the VAE temporal stride of 4). When
|
||||
> you change `num_frames`, pass a matching `chunk_partition` summing to `(num_frames - 1) // 4 + 1`,
|
||||
> otherwise the pipeline raises a `ValueError`.
|
||||
|
||||
<hfoptions id="anyflow-far">
|
||||
<hfoption id="t2v">
|
||||
|
||||
@@ -125,11 +125,11 @@ Causal pipeline 用同一个蒸馏模型支持三种任务模式,**通过 `vid
|
||||
Context tensor 的帧数必须满足 `T = 4n + 1`,跟 VAE 时间步长对齐。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> FAR pipeline 是分块 (chunk) rollout,`num_frames` 必须配合 chunk 调度。默认
|
||||
> `chunk_partition=[1, 3, 3, 3, 3, 3, 3, 2]`(求和 21)对应发布 checkpoint 的标准 `num_frames=81`
|
||||
> (21 = (81 − 1) // 4 + 1)。改 `num_frames` 时**必须**显式传匹配的 `chunk_partition`,使其求和等于
|
||||
> `(num_frames - 1) // 4 + 1`,否则 pipeline 会抛 `AssertionError`。比如 `num_frames=33` 对应 9 个 latent
|
||||
> 帧,可用 `chunk_partition=[1, 4, 4]`。
|
||||
> FAR pipeline 是分块 (chunk) rollout,`num_frames` 必须配合 chunk 调度。发布的 checkpoint 在
|
||||
> transformer config 里写入 `chunk_partition=[1, 3, 3, 3, 3, 3, 3, 2]`(求和 21),对应标准
|
||||
> `num_frames=81`(21 = (81 − 1) // 4 + 1)。改 `num_frames` 时**必须**显式传匹配的 `chunk_partition`,
|
||||
> 使其求和等于 `(num_frames - 1) // 4 + 1`,否则 pipeline 会抛 `ValueError`。比如 `num_frames=33` 对应
|
||||
> 9 个 latent 帧,可用 `chunk_partition=[1, 4, 4]`。
|
||||
|
||||
```py
|
||||
import numpy as np
|
||||
|
||||
@@ -57,13 +57,21 @@ VARIANTS = {
|
||||
"AnyFlow-FAR-Wan2.1-1.3B-Diffusers": {
|
||||
"base_model": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||
"transformer_cls": AnyFlowFARTransformer3DModel,
|
||||
"transformer_kwargs": {"full_chunk_limit": 3, "compressed_patch_size": [1, 4, 4]},
|
||||
"transformer_kwargs": {
|
||||
"full_chunk_limit": 3,
|
||||
"compressed_patch_size": [1, 4, 4],
|
||||
"chunk_partition": [1, 3, 3, 3, 3, 3, 3, 2],
|
||||
},
|
||||
"pipeline_cls": AnyFlowFARPipeline,
|
||||
},
|
||||
"AnyFlow-FAR-Wan2.1-14B-Diffusers": {
|
||||
"base_model": "Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
||||
"transformer_cls": AnyFlowFARTransformer3DModel,
|
||||
"transformer_kwargs": {"full_chunk_limit": 3, "compressed_patch_size": [1, 4, 4]},
|
||||
"transformer_kwargs": {
|
||||
"full_chunk_limit": 3,
|
||||
"compressed_patch_size": [1, 4, 4],
|
||||
"chunk_partition": [1, 3, 3, 3, 3, 3, 3, 2],
|
||||
},
|
||||
"pipeline_cls": AnyFlowFARPipeline,
|
||||
},
|
||||
"AnyFlow-Wan2.1-T2V-1.3B-Diffusers": {
|
||||
|
||||
@@ -1007,11 +1007,11 @@ class AnyFlowFARTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin, Fr
|
||||
Mixing gate between source-timestep and delta-timestep embeddings.
|
||||
deltatime_type (`str`, defaults to `'r'`):
|
||||
Either ``"r"`` (delta is the target timestep) or ``"t-r"`` (delta is the absolute interval).
|
||||
|
||||
.. note::
|
||||
``chunk_partition`` is **not** a model config field — it is a per-call argument passed to :meth:`forward`.
|
||||
Different inference setups (varying ``num_frames`` or full-vs-compressed schedules) therefore do not require
|
||||
separate checkpoints.
|
||||
chunk_partition (`Tuple[int, ...]`, defaults to `(1, 3, 3, 3, 3, 3, 3, 2)`):
|
||||
Default per-chunk frame counts used by the pipeline. The released NVIDIA AnyFlow-FAR checkpoints
|
||||
target ``num_frames=81`` (21 latent frames at VAE temporal stride 4) split as
|
||||
``1 + 3*6 + 2``. A different ``num_frames`` requires a matching ``chunk_partition`` override
|
||||
passed to :meth:`AnyFlowFARPipeline.__call__` (and likewise to :meth:`forward`).
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = True
|
||||
@@ -1040,6 +1040,7 @@ class AnyFlowFARTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin, Fr
|
||||
rope_max_seq_len: int = 1024,
|
||||
gate_value: float = 0.25,
|
||||
deltatime_type: str = "r",
|
||||
chunk_partition: Tuple[int, ...] = (1, 3, 3, 3, 3, 3, 3, 2),
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
|
||||
@@ -129,11 +129,6 @@ class AnyFlowFARPipeline(DiffusionPipeline, WanLoraLoaderMixin):
|
||||
model_cpu_offload_seq = "text_encoder->transformer->vae"
|
||||
_callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
|
||||
|
||||
# Default chunk partition for the released NVIDIA AnyFlow-FAR checkpoints (81 frames at the diffusers
|
||||
# VAE temporal stride of 4 → 21 latent frames split into 1 + 3*6 + 2 = [1, 3, 3, 3, 3, 3, 3, 2]). Override
|
||||
# via the ``chunk_partition`` argument to ``__call__`` for other frame counts.
|
||||
default_chunk_partition: List[int] = [1, 3, 3, 3, 3, 3, 3, 2]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: AutoTokenizer,
|
||||
@@ -547,9 +542,9 @@ class AnyFlowFARPipeline(DiffusionPipeline, WanLoraLoaderMixin):
|
||||
use_kv_cache (`bool`, defaults to `True`):
|
||||
Reuse the FAR attention KV cache across causal chunks. Disable only for debugging.
|
||||
chunk_partition (`List[int]`, *optional*):
|
||||
Per-chunk frame counts. Defaults to `default_chunk_partition` (matched to the released 81-frame
|
||||
checkpoints). When you change `num_frames`, supply a `chunk_partition` that sums to `(num_frames - 1)
|
||||
// vae_scale_factor_temporal + 1`.
|
||||
Per-chunk frame counts. Defaults to `self.transformer.config.chunk_partition` (matched to the
|
||||
released 81-frame checkpoints). When you change `num_frames`, supply a `chunk_partition` that
|
||||
sums to `(num_frames - 1) // vae_scale_factor_temporal + 1`.
|
||||
|
||||
Examples:
|
||||
|
||||
@@ -638,7 +633,7 @@ class AnyFlowFARPipeline(DiffusionPipeline, WanLoraLoaderMixin):
|
||||
video_latents = self.encode_video(video, height=height, width=width)
|
||||
|
||||
if chunk_partition is None:
|
||||
chunk_partition = list(self.default_chunk_partition)
|
||||
chunk_partition = list(self.transformer.config.chunk_partition)
|
||||
if init_latents.shape[1] != sum(chunk_partition):
|
||||
raise ValueError(
|
||||
f"chunk_partition={chunk_partition} sums to {sum(chunk_partition)}, but the input latent "
|
||||
|
||||
@@ -90,6 +90,7 @@ class AnyFlowFARPipelineFastTests(PipelineTesterMixin, unittest.TestCase):
|
||||
rope_max_seq_len=32,
|
||||
gate_value=0.25,
|
||||
deltatime_type="r",
|
||||
chunk_partition=(1, 1, 1),
|
||||
)
|
||||
|
||||
components = {
|
||||
@@ -106,8 +107,8 @@ class AnyFlowFARPipelineFastTests(PipelineTesterMixin, unittest.TestCase):
|
||||
generator = torch.manual_seed(seed)
|
||||
else:
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
# num_frames=9 -> 3 latent frames (VAE temporal stride 4); use a matching
|
||||
# chunk_partition so the FAR pipeline's pre-flight assertion passes.
|
||||
# num_frames=9 -> 3 latent frames (VAE temporal stride 4); the transformer config above
|
||||
# has chunk_partition=(1, 1, 1) (sum 3) baked in, so __call__ picks it up automatically.
|
||||
inputs = {
|
||||
"prompt": "dance monkey",
|
||||
"negative_prompt": "negative",
|
||||
@@ -119,7 +120,6 @@ class AnyFlowFARPipelineFastTests(PipelineTesterMixin, unittest.TestCase):
|
||||
"num_frames": 9,
|
||||
"max_sequence_length": 16,
|
||||
"output_type": "pt",
|
||||
"chunk_partition": [1, 1, 1],
|
||||
}
|
||||
return inputs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user