mirror of
https://github.com/huggingface/diffusers.git
synced 2026-10-24 20:59:27 +03:00
771779849b
* migrate imports * migrate stable diffusion adapter tests * up * move last remaining bits. * simplify test policy wording * move around * move flux ip adapter testing stuff to flux module of test * fix(ci): harden GitHub Actions workflows (#14669) (#14787) fix(ci): harden workflow files flagged on #14669 Co-authored-by: hf-security-analysis[bot] <265538906+hf-security-analysis[bot]@users.noreply.github.com> * clarify comment. * remove name from batch_input --------- Co-authored-by: hf-security-analysis[bot] <265538906+hf-security-analysis[bot]@users.noreply.github.com>
241 lines
11 KiB
Python
241 lines
11 KiB
Python
# coding=utf-8
|
|
# Copyright 2026 HuggingFace Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import inspect
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from diffusers.loaders import IPAdapterMixin
|
|
|
|
from ...testing_utils import assert_tensors_close, is_ip_adapter, torch_device
|
|
from .common import BasePipelineOutputMixin
|
|
|
|
|
|
@is_ip_adapter
|
|
class IPAdapterTesterMixin(BasePipelineOutputMixin):
|
|
"""IP-Adapter tests for UNet pipelines that load adapters through the standard `IPAdapterMixin` API.
|
|
|
|
Compose it with a `BasePipelineTesterConfig` subclass in its own test class, separate from the
|
|
`PipelineTesterMixin` one. Pipelines whose IP-Adapter API differs use a mixin of their own.
|
|
"""
|
|
|
|
def _get_dummy_image_embeds(self, cross_attention_dim: int = 32):
|
|
return torch.randn((2, 1, cross_attention_dim), device=torch_device)
|
|
|
|
def _get_dummy_faceid_image_embeds(self, cross_attention_dim: int = 32):
|
|
return torch.randn((2, 1, 1, cross_attention_dim), device=torch_device)
|
|
|
|
def _get_dummy_masks(self, input_size: int = 64):
|
|
masks = torch.zeros((1, 1, input_size, input_size), device=torch_device)
|
|
masks[0, :, :, : input_size // 2] = 1
|
|
return masks
|
|
|
|
def _get_ip_adapter_inputs(self):
|
|
inputs = self.get_dummy_inputs()
|
|
parameters = inspect.signature(self.pipeline_class.__call__).parameters
|
|
if "image" in parameters and "strength" in parameters:
|
|
inputs["num_inference_steps"] = 4
|
|
inputs["return_dict"] = False
|
|
return inputs
|
|
|
|
def _load_ip_adapters(self, pipe, num_adapters=1, faceid=False):
|
|
# The state dict builders are imported here rather than at module scope: they live in a model test module
|
|
# that calls `enable_full_determinism()` on import, which would otherwise flip that global for every test
|
|
# collected alongside this directory.
|
|
from ...models.unets.test_models_unet_2d_condition import (
|
|
create_ip_adapter_faceid_state_dict,
|
|
create_ip_adapter_state_dict,
|
|
)
|
|
|
|
create_state_dict = create_ip_adapter_faceid_state_dict if faceid else create_ip_adapter_state_dict
|
|
state_dicts = [create_state_dict(pipe.unet) for _ in range(num_adapters)]
|
|
|
|
# Load through the pipeline's public IP-Adapter API. `image_encoder_folder=None` skips fetching a CLIP image
|
|
# encoder since these tests feed pre-computed `ip_adapter_image_embeds` directly.
|
|
pipe.load_ip_adapter(
|
|
state_dicts,
|
|
subfolder=[""] * num_adapters,
|
|
weight_name=[""] * num_adapters,
|
|
image_encoder_folder=None,
|
|
)
|
|
|
|
def test_pipeline_signature(self):
|
|
parameters = inspect.signature(self.pipeline_class.__call__).parameters
|
|
|
|
assert issubclass(self.pipeline_class, IPAdapterMixin)
|
|
assert "ip_adapter_image" in parameters, (
|
|
"`ip_adapter_image` argument must be supported by the `__call__` method"
|
|
)
|
|
assert "ip_adapter_image_embeds" in parameters, (
|
|
"`ip_adapter_image_embeds` argument must be supported by the `__call__` method"
|
|
)
|
|
|
|
def test_ip_adapter(self, expected_max_diff: float = 1e-4):
|
|
r"""Tests for IP-Adapter.
|
|
|
|
The following scenarios are tested:
|
|
- Single IP-Adapter with scale=0 should produce same output as no IP-Adapter.
|
|
- Multi IP-Adapter with scale=0 should produce same output as no IP-Adapter.
|
|
- Single IP-Adapter with scale!=0 should produce different output compared to no IP-Adapter.
|
|
- Multi IP-Adapter with scale!=0 should produce different output compared to no IP-Adapter.
|
|
"""
|
|
pipe = self.get_pipeline().to(torch_device)
|
|
cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32)
|
|
|
|
# forward pass without ip adapter
|
|
output_without_adapter = pipe(**self._get_ip_adapter_inputs())[0]
|
|
|
|
# 1. Single IP-Adapter test cases
|
|
self._load_ip_adapters(pipe)
|
|
|
|
# forward pass with single ip adapter, but scale=0 which should have no effect
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)]
|
|
pipe.set_ip_adapter_scale(0.0)
|
|
output_without_adapter_scale = pipe(**inputs)[0]
|
|
|
|
# forward pass with single ip adapter, but with scale of adapter weights
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)]
|
|
pipe.set_ip_adapter_scale(42.0)
|
|
output_with_adapter_scale = pipe(**inputs)[0]
|
|
|
|
assert_tensors_close(
|
|
output_without_adapter_scale,
|
|
output_without_adapter,
|
|
atol=expected_max_diff,
|
|
msg="Output without ip-adapter must be same as normal inference",
|
|
)
|
|
max_diff_with_adapter_scale = (output_with_adapter_scale - output_without_adapter).abs().max()
|
|
assert max_diff_with_adapter_scale > 1e-2, "Output with ip-adapter must be different from normal inference"
|
|
|
|
# 2. Multi IP-Adapter test cases
|
|
self._load_ip_adapters(pipe, num_adapters=2)
|
|
|
|
# forward pass with multi ip adapter, but scale=0 which should have no effect
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] * 2
|
|
pipe.set_ip_adapter_scale([0.0, 0.0])
|
|
output_without_multi_adapter_scale = pipe(**inputs)[0]
|
|
|
|
# forward pass with multi ip adapter, but with scale of adapter weights
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] * 2
|
|
pipe.set_ip_adapter_scale([42.0, 42.0])
|
|
output_with_multi_adapter_scale = pipe(**inputs)[0]
|
|
|
|
assert_tensors_close(
|
|
output_without_multi_adapter_scale,
|
|
output_without_adapter,
|
|
atol=expected_max_diff,
|
|
msg="Output without multi-ip-adapter must be same as normal inference",
|
|
)
|
|
max_diff_with_multi_adapter_scale = (output_with_multi_adapter_scale - output_without_adapter).abs().max()
|
|
assert max_diff_with_multi_adapter_scale > 1e-2, (
|
|
"Output with multi-ip-adapter scale must be different from normal inference"
|
|
)
|
|
|
|
def test_ip_adapter_cfg(self):
|
|
if "guidance_scale" not in inspect.signature(self.pipeline_class.__call__).parameters:
|
|
pytest.skip(
|
|
f"Skipping test because `guidance_scale` wasn't found in the args accepted in {self.pipeline_class}'s call."
|
|
)
|
|
|
|
pipe = self.get_pipeline().to(torch_device)
|
|
cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32)
|
|
|
|
self._load_ip_adapters(pipe)
|
|
pipe.set_ip_adapter_scale(1.0)
|
|
|
|
# forward pass with CFG not applied
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)[0].unsqueeze(0)]
|
|
inputs["guidance_scale"] = 1.0
|
|
out_no_cfg = pipe(**inputs)[0]
|
|
|
|
# forward pass with CFG applied
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)]
|
|
inputs["guidance_scale"] = 7.5
|
|
out_cfg = pipe(**inputs)[0]
|
|
|
|
assert out_cfg.shape == out_no_cfg.shape
|
|
|
|
def test_ip_adapter_masks(self, expected_max_diff: float = 1e-4):
|
|
pipe = self.get_pipeline().to(torch_device)
|
|
cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32)
|
|
sample_size = pipe.unet.config.get("sample_size", 32)
|
|
block_out_channels = pipe.vae.config.get("block_out_channels", [128, 256, 512, 512])
|
|
input_size = sample_size * (2 ** (len(block_out_channels) - 1))
|
|
|
|
# forward pass without ip adapter
|
|
output_without_adapter = pipe(**self._get_ip_adapter_inputs())[0]
|
|
|
|
self._load_ip_adapters(pipe)
|
|
|
|
# forward pass with single ip adapter and masks, but scale=0 which should have no effect
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)]
|
|
inputs["cross_attention_kwargs"] = {"ip_adapter_masks": [self._get_dummy_masks(input_size)]}
|
|
pipe.set_ip_adapter_scale(0.0)
|
|
output_without_adapter_scale = pipe(**inputs)[0]
|
|
|
|
# forward pass with single ip adapter and masks, but with scale of adapter weights
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)]
|
|
inputs["cross_attention_kwargs"] = {"ip_adapter_masks": [self._get_dummy_masks(input_size)]}
|
|
pipe.set_ip_adapter_scale(42.0)
|
|
output_with_adapter_scale = pipe(**inputs)[0]
|
|
|
|
assert_tensors_close(
|
|
output_without_adapter_scale,
|
|
output_without_adapter,
|
|
atol=expected_max_diff,
|
|
msg="Output without ip-adapter must be same as normal inference",
|
|
)
|
|
max_diff_with_adapter_scale = (output_with_adapter_scale - output_without_adapter).abs().max()
|
|
assert max_diff_with_adapter_scale > 1e-3, "Output with ip-adapter must be different from normal inference"
|
|
|
|
def test_ip_adapter_faceid(self, expected_max_diff: float = 1e-4):
|
|
pipe = self.get_pipeline().to(torch_device)
|
|
cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32)
|
|
|
|
# forward pass without ip adapter
|
|
output_without_adapter = pipe(**self._get_ip_adapter_inputs())[0]
|
|
|
|
self._load_ip_adapters(pipe, faceid=True)
|
|
|
|
# forward pass with single ip adapter, but scale=0 which should have no effect
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_faceid_image_embeds(cross_attention_dim)]
|
|
pipe.set_ip_adapter_scale(0.0)
|
|
output_without_adapter_scale = pipe(**inputs)[0]
|
|
|
|
# forward pass with single ip adapter, but with scale of adapter weights
|
|
inputs = self._get_ip_adapter_inputs()
|
|
inputs["ip_adapter_image_embeds"] = [self._get_dummy_faceid_image_embeds(cross_attention_dim)]
|
|
pipe.set_ip_adapter_scale(42.0)
|
|
output_with_adapter_scale = pipe(**inputs)[0]
|
|
|
|
assert_tensors_close(
|
|
output_without_adapter_scale,
|
|
output_without_adapter,
|
|
atol=expected_max_diff,
|
|
msg="Output without ip-adapter must be same as normal inference",
|
|
)
|
|
max_diff_with_adapter_scale = (output_with_adapter_scale - output_without_adapter).abs().max()
|
|
assert max_diff_with_adapter_scale > 1e-3, "Output with ip-adapter must be different from normal inference"
|