diff --git a/.gitignore b/.gitignore index 15617d5fdc..a55026febd 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,9 @@ dmypy.json .vs .vscode +# Cursor +.cursor + # Pycharm .idea diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index fb4fdf2098..848e38079e 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -49,7 +49,7 @@ isExpanded: false sections: - local: using-diffusers/weighted_prompts - title: Prompt techniques + title: Prompting - local: using-diffusers/create_a_server title: Create a server - local: using-diffusers/batched_inference diff --git a/docs/source/en/api/loaders/ip_adapter.md b/docs/source/en/api/loaders/ip_adapter.md index c2c45bee10..508e6d4ee6 100644 --- a/docs/source/en/api/loaders/ip_adapter.md +++ b/docs/source/en/api/loaders/ip_adapter.md @@ -15,7 +15,7 @@ specific language governing permissions and limitations under the License. [IP-Adapter](https://hf.co/papers/2308.06721) is a lightweight adapter that enables prompting a diffusion model with an image. This method decouples the cross-attention layers of the image and text features. The image features are generated from an image encoder. > [!TIP] -> Learn how to load an IP-Adapter checkpoint and image in the IP-Adapter [loading](../../using-diffusers/loading_adapters#ip-adapter) guide, and you can see how to use it in the [usage](../../using-diffusers/ip_adapter) guide. +> Learn how to load and use an IP-Adapter checkpoint and image in the [IP-Adapter](../../using-diffusers/ip_adapter) guide,. ## IPAdapterMixin diff --git a/docs/source/en/api/loaders/lora.md b/docs/source/en/api/loaders/lora.md index bf22a32d74..b1d1ffb634 100644 --- a/docs/source/en/api/loaders/lora.md +++ b/docs/source/en/api/loaders/lora.md @@ -34,7 +34,7 @@ LoRA is a fast and lightweight training method that inserts and trains a signifi - [`LoraBaseMixin`] provides a base class with several utility methods to fuse, unfuse, unload, LoRAs and more. > [!TIP] -> To learn more about how to load LoRA weights, see the [LoRA](../../using-diffusers/loading_adapters#lora) loading guide. +> To learn more about how to load LoRA weights, see the [LoRA](../../tutorials/using_peft_for_inference) loading guide. ## LoraBaseMixin diff --git a/docs/source/en/api/loaders/peft.md b/docs/source/en/api/loaders/peft.md index 5508509c88..c514766dd8 100644 --- a/docs/source/en/api/loaders/peft.md +++ b/docs/source/en/api/loaders/peft.md @@ -12,7 +12,7 @@ specific language governing permissions and limitations under the License. # PEFT -Diffusers supports loading adapters such as [LoRA](../../using-diffusers/loading_adapters) with the [PEFT](https://huggingface.co/docs/peft/index) library with the [`~loaders.peft.PeftAdapterMixin`] class. This allows modeling classes in Diffusers like [`UNet2DConditionModel`], [`SD3Transformer2DModel`] to operate with an adapter. +Diffusers supports loading adapters such as [LoRA](../../tutorials/using_peft_for_inference) with the [PEFT](https://huggingface.co/docs/peft/index) library with the [`~loaders.peft.PeftAdapterMixin`] class. This allows modeling classes in Diffusers like [`UNet2DConditionModel`], [`SD3Transformer2DModel`] to operate with an adapter. > [!TIP] > Refer to the [Inference with PEFT](../../tutorials/using_peft_for_inference.md) tutorial for an overview of how to use PEFT in Diffusers for inference. diff --git a/docs/source/en/api/loaders/textual_inversion.md b/docs/source/en/api/loaders/textual_inversion.md index 2cb54ce4ea..5e8bfac255 100644 --- a/docs/source/en/api/loaders/textual_inversion.md +++ b/docs/source/en/api/loaders/textual_inversion.md @@ -17,7 +17,7 @@ Textual Inversion is a training method for personalizing models by learning new [`TextualInversionLoaderMixin`] provides a function for loading Textual Inversion embeddings from Diffusers and Automatic1111 into the text encoder and loading a special token to activate the embeddings. > [!TIP] -> To learn more about how to load Textual Inversion embeddings, see the [Textual Inversion](../../using-diffusers/loading_adapters#textual-inversion) loading guide. +> To learn more about how to load Textual Inversion embeddings, see the [Textual Inversion](../../using-diffusers/textual_inversion_inference) loading guide. ## TextualInversionLoaderMixin diff --git a/docs/source/en/api/loaders/transformer_sd3.md b/docs/source/en/api/loaders/transformer_sd3.md index cc3ec0da14..2c8b81b59c 100644 --- a/docs/source/en/api/loaders/transformer_sd3.md +++ b/docs/source/en/api/loaders/transformer_sd3.md @@ -17,7 +17,7 @@ This class is useful when *only* loading weights into a [`SD3Transformer2DModel` The [`SD3Transformer2DLoadersMixin`] class currently only loads IP-Adapter weights, but will be used in the future to save weights and load LoRAs. > [!TIP] -> To learn more about how to load LoRA weights, see the [LoRA](../../using-diffusers/loading_adapters#lora) loading guide. +> To learn more about how to load LoRA weights, see the [LoRA](../../tutorials/using_peft_for_inference) loading guide. ## SD3Transformer2DLoadersMixin diff --git a/docs/source/en/api/loaders/unet.md b/docs/source/en/api/loaders/unet.md index 7450e03e58..50d210bbf5 100644 --- a/docs/source/en/api/loaders/unet.md +++ b/docs/source/en/api/loaders/unet.md @@ -17,7 +17,7 @@ Some training methods - like LoRA and Custom Diffusion - typically target the UN The [`UNet2DConditionLoadersMixin`] class provides functions for loading and saving weights, fusing and unfusing LoRAs, disabling and enabling LoRAs, and setting and deleting adapters. > [!TIP] -> To learn more about how to load LoRA weights, see the [LoRA](../../using-diffusers/loading_adapters#lora) loading guide. +> To learn more about how to load LoRA weights, see the [LoRA](../../tutorials/using_peft_for_inference) guide. ## UNet2DConditionLoadersMixin diff --git a/docs/source/en/api/pipelines/flux.md b/docs/source/en/api/pipelines/flux.md index 1a89de98e4..358b8139c7 100644 --- a/docs/source/en/api/pipelines/flux.md +++ b/docs/source/en/api/pipelines/flux.md @@ -418,7 +418,7 @@ When unloading the Control LoRA weights, call `pipe.unload_lora_weights(reset_to ## IP-Adapter > [!TIP] -> Check out [IP-Adapter](../../../using-diffusers/ip_adapter) to learn more about how IP-Adapters work. +> Check out [IP-Adapter](../../using-diffusers/ip_adapter) to learn more about how IP-Adapters work. An IP-Adapter lets you prompt Flux with images, in addition to the text prompt. This is especially useful when describing complex concepts that are difficult to articulate through text alone and you have reference images. diff --git a/docs/source/en/api/pipelines/hidream.md b/docs/source/en/api/pipelines/hidream.md index acfcef93e0..add4ad3132 100644 --- a/docs/source/en/api/pipelines/hidream.md +++ b/docs/source/en/api/pipelines/hidream.md @@ -21,7 +21,7 @@ ## Available models -The following models are available for the [`HiDreamImagePipeline`](text-to-image) pipeline: +The following models are available for the [`HiDreamImagePipeline`] pipeline: | Model name | Description | |:---|:---| diff --git a/docs/source/en/api/pipelines/overview.md b/docs/source/en/api/pipelines/overview.md index ce883931df..22fcf560ea 100644 --- a/docs/source/en/api/pipelines/overview.md +++ b/docs/source/en/api/pipelines/overview.md @@ -32,7 +32,7 @@ The table below lists all the pipelines currently available in 🤗 Diffusers an | [Attend-and-Excite](attend_and_excite) | text2image | | [AudioLDM](audioldm) | text2audio | | [AudioLDM2](audioldm2) | text2audio | -| [AuraFlow](auraflow) | text2image | +| [AuraFlow](aura_flow) | text2image | | [BLIP Diffusion](blip_diffusion) | text2image | | [Bria 3.2](bria_3_2) | text2image | | [CogVideoX](cogvideox) | text2video | diff --git a/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_3.md b/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_3.md index 3c49df101c..4c7e5b1073 100644 --- a/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_3.md +++ b/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_3.md @@ -271,7 +271,7 @@ Check out the full script [here](https://gist.github.com/sayakpaul/508d89d7aad4f Quantization helps reduce the memory requirements of very large models by storing model weights in a lower precision data type. However, quantization may have varying impact on video quality depending on the video model. -Refer to the [Quantization](../../quantization/overview) overview to learn more about supported quantization backends and selecting a quantization backend that supports your use case. The example below demonstrates how to load a quantized [`StableDiffusion3Pipeline`] for inference with bitsandbytes. +Refer to the [Quantization](../../../quantization/overview) overview to learn more about supported quantization backends and selecting a quantization backend that supports your use case. The example below demonstrates how to load a quantized [`StableDiffusion3Pipeline`] for inference with bitsandbytes. ```py import torch diff --git a/docs/source/en/api/pipelines/stable_diffusion/svd.md b/docs/source/en/api/pipelines/stable_diffusion/svd.md index 0c33c06007..a00dd3ef6d 100644 --- a/docs/source/en/api/pipelines/stable_diffusion/svd.md +++ b/docs/source/en/api/pipelines/stable_diffusion/svd.md @@ -29,7 +29,7 @@ The abstract from the paper is: Video generation is memory-intensive and one way to reduce your memory usage is to set `enable_forward_chunking` on the pipeline's UNet so you don't run the entire feedforward layer at once. Breaking it up into chunks in a loop is more efficient. -Check out the [Text or image-to-video](text-img2vid) guide for more details about how certain parameters can affect video generation and how to optimize inference by reducing memory usage. +Check out the [Text or image-to-video](../../../using-diffusers/text-img2vid) guide for more details about how certain parameters can affect video generation and how to optimize inference by reducing memory usage. ## StableVideoDiffusionPipeline diff --git a/docs/source/en/api/pipelines/text_to_video.md b/docs/source/en/api/pipelines/text_to_video.md index d7c37be637..d9f6d8e722 100644 --- a/docs/source/en/api/pipelines/text_to_video.md +++ b/docs/source/en/api/pipelines/text_to_video.md @@ -172,7 +172,7 @@ Here are some sample outputs: Video generation is memory-intensive and one way to reduce your memory usage is to set `enable_forward_chunking` on the pipeline's UNet so you don't run the entire feedforward layer at once. Breaking it up into chunks in a loop is more efficient. -Check out the [Text or image-to-video](text-img2vid) guide for more details about how certain parameters can affect video generation and how to optimize inference by reducing memory usage. +Check out the [Text or image-to-video](../../using-diffusers/text-img2vid) guide for more details about how certain parameters can affect video generation and how to optimize inference by reducing memory usage. > [!TIP] > Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers) to learn how to explore the tradeoff between scheduler speed and quality, and see the [reuse components across pipelines](../../using-diffusers/loading#reuse-a-pipeline) section to learn how to efficiently load the same components into multiple pipelines. diff --git a/docs/source/en/training/dreambooth.md b/docs/source/en/training/dreambooth.md index 81ed09c9d0..2302739a0e 100644 --- a/docs/source/en/training/dreambooth.md +++ b/docs/source/en/training/dreambooth.md @@ -548,4 +548,4 @@ Training the DeepFloyd IF model can be challenging, but here are some tips that Congratulations on training your DreamBooth model! To learn more about how to use your new model, the following guide may be helpful: -- Learn how to [load a DreamBooth](../using-diffusers/loading_adapters) model for inference if you trained your model with LoRA. \ No newline at end of file +- Learn how to [load a DreamBooth](../using-diffusers/dreambooth) model for inference if you trained your model with LoRA. \ No newline at end of file diff --git a/docs/source/en/training/lcm_distill.md b/docs/source/en/training/lcm_distill.md index 232f2eceed..4750f15036 100644 --- a/docs/source/en/training/lcm_distill.md +++ b/docs/source/en/training/lcm_distill.md @@ -75,7 +75,7 @@ accelerate launch train_lcm_distill_sd_wds.py \ Most of the parameters are identical to the parameters in the [Text-to-image](text2image#script-parameters) training guide, so you'll focus on the parameters that are relevant to latent consistency distillation in this guide. - `--pretrained_teacher_model`: the path to a pretrained latent diffusion model to use as the teacher model -- `--pretrained_vae_model_name_or_path`: path to a pretrained VAE; the SDXL VAE is known to suffer from numerical instability, so this parameter allows you to specify an alternative VAE (like this [VAE]((https://huggingface.co/madebyollin/sdxl-vae-fp16-fix)) by madebyollin which works in fp16) +- `--pretrained_vae_model_name_or_path`: path to a pretrained VAE; the SDXL VAE is known to suffer from numerical instability, so this parameter allows you to specify an alternative VAE (like this [VAE](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix)) by madebyollin which works in fp16) - `--w_min` and `--w_max`: the minimum and maximum guidance scale values for guidance scale sampling - `--num_ddim_timesteps`: the number of timesteps for DDIM sampling - `--loss_type`: the type of loss (L2 or Huber) to calculate for latent consistency distillation; Huber loss is generally preferred because it's more robust to outliers @@ -245,5 +245,5 @@ The SDXL training script is discussed in more detail in the [SDXL training](sdxl Congratulations on distilling a LCM model! To learn more about LCM, the following may be helpful: -- Learn how to use [LCMs for inference](../using-diffusers/lcm) for text-to-image, image-to-image, and with LoRA checkpoints. +- Learn how to use [LCMs for inference](../using-diffusers/inference_with_lcm) for text-to-image, image-to-image, and with LoRA checkpoints. - Read the [SDXL in 4 steps with Latent Consistency LoRAs](https://huggingface.co/blog/lcm_lora) blog post to learn more about SDXL LCM-LoRA's for super fast inference, quality comparisons, benchmarks, and more. diff --git a/docs/source/en/training/lora.md b/docs/source/en/training/lora.md index 45a234b76a..efb170e329 100644 --- a/docs/source/en/training/lora.md +++ b/docs/source/en/training/lora.md @@ -198,5 +198,5 @@ image = pipeline("A naruto with blue eyes").images[0] Congratulations on training a new model with LoRA! To learn more about how to use your new model, the following guides may be helpful: -- Learn how to [load different LoRA formats](../using-diffusers/loading_adapters#LoRA) trained using community trainers like Kohya and TheLastBen. +- Learn how to [load different LoRA formats](../tutorials/using_peft_for_inference) trained using community trainers like Kohya and TheLastBen. - Learn how to use and [combine multiple LoRA's](../tutorials/using_peft_for_inference) with PEFT for inference. diff --git a/docs/source/en/training/text2image.md b/docs/source/en/training/text2image.md index a9327457c7..d11e55e910 100644 --- a/docs/source/en/training/text2image.md +++ b/docs/source/en/training/text2image.md @@ -178,5 +178,5 @@ image.save("yoda-naruto.png") Congratulations on training your own text-to-image model! To learn more about how to use your new model, the following guides may be helpful: -- Learn how to [load LoRA weights](../using-diffusers/loading_adapters#LoRA) for inference if you trained your model with LoRA. +- Learn how to [load LoRA weights](../tutorials/using_peft_for_inference) for inference if you trained your model with LoRA. - Learn more about how certain parameters like guidance scale or techniques such as prompt weighting can help you control inference in the [Text-to-image](../using-diffusers/conditional_image_generation) task guide. diff --git a/docs/source/en/training/text_inversion.md b/docs/source/en/training/text_inversion.md index 0b540107e9..4912b6730a 100644 --- a/docs/source/en/training/text_inversion.md +++ b/docs/source/en/training/text_inversion.md @@ -203,5 +203,4 @@ image.save("cat-train.png") Congratulations on training your own Textual Inversion model! 🎉 To learn more about how to use your new model, the following guides may be helpful: -- Learn how to [load Textual Inversion embeddings](../using-diffusers/loading_adapters) and also use them as negative embeddings. -- Learn how to use [Textual Inversion](textual_inversion_inference) for inference with Stable Diffusion 1/2 and Stable Diffusion XL. \ No newline at end of file +- Learn how to [load Textual Inversion embeddings](../using-diffusers/textual_inversion_inference) and also use them as negative embeddings. \ No newline at end of file diff --git a/docs/source/en/using-diffusers/batched_inference.md b/docs/source/en/using-diffusers/batched_inference.md index b5e55c27ca..cdb16ac121 100644 --- a/docs/source/en/using-diffusers/batched_inference.md +++ b/docs/source/en/using-diffusers/batched_inference.md @@ -16,24 +16,24 @@ Batch inference processes multiple prompts at a time to increase throughput. It The downside is increased latency because you must wait for the entire batch to complete, and more GPU memory is required for large batches. - - - -For text-to-image, pass a list of prompts to the pipeline. +For text-to-image, pass a list of prompts to the pipeline and for image-to-image, pass a list of images and prompts to the pipeline. The example below demonstrates batched text-to-image inference. ```py import torch +import matplotlib.pyplot as plt from diffusers import DiffusionPipeline pipeline = DiffusionPipeline.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", - torch_dtype=torch.float16 -).to("cuda") + torch_dtype=torch.float16, + device_map="cuda" +) prompts = [ - "cinematic photo of A beautiful sunset over mountains, 35mm photograph, film, professional, 4k, highly detailed", - "cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain", - "pixel-art a cozy coffee shop interior, low-res, blocky, pixel art style, 8-bit graphics" + "Cinematic shot of a cozy coffee shop interior, warm pastel light streaming through a window where a cat rests. Shallow depth of field, glowing cups in soft focus, dreamy lofi-inspired mood, nostalgic tones, framed like a quiet film scene.", + "Polaroid-style photograph of a cozy coffee shop interior, bathed in warm pastel light. A cat sits on the windowsill near steaming mugs. Soft, slightly faded tones and dreamy blur evoke nostalgia, a lofi mood, and the intimate, imperfect charm of instant film.", + "Soft watercolor illustration of a cozy coffee shop interior, pastel washes of color filling the space. A cat rests peacefully on the windowsill as warm light glows through. Gentle brushstrokes create a dreamy, lofi-inspired atmosphere with whimsical textures and nostalgic calm.", + "Isometric pixel-art illustration of a cozy coffee shop interior in detailed 8-bit style. Warm pastel light fills the space as a cat rests on the windowsill. Blocky furniture and tiny mugs add charm, low-res retro graphics enhance the nostalgic, lofi-inspired game aesthetic." ] images = pipeline( @@ -52,6 +52,10 @@ plt.tight_layout() plt.show() ``` +
+ +
+ To generate multiple variations of one prompt, use the `num_images_per_prompt` argument. ```py @@ -61,11 +65,18 @@ from diffusers import DiffusionPipeline pipeline = DiffusionPipeline.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", - torch_dtype=torch.float16 -).to("cuda") + torch_dtype=torch.float16, + device_map="cuda" +) + +prompt=""" +Isometric pixel-art illustration of a cozy coffee shop interior in detailed 8-bit style. Warm pastel light fills the +space as a cat rests on the windowsill. Blocky furniture and tiny mugs add charm, low-res retro graphics enhance the +nostalgic, lofi-inspired game aesthetic. +""" images = pipeline( - prompt="pixel-art a cozy coffee shop interior, low-res, blocky, pixel art style, 8-bit graphics", + prompt=prompt, num_images_per_prompt=4 ).images @@ -81,6 +92,10 @@ plt.tight_layout() plt.show() ``` +
+ +
+ Combine both approaches to generate different variations of different prompts. ```py @@ -89,7 +104,7 @@ images = pipeline( num_images_per_prompt=2, ).images -fig, axes = plt.subplots(2, 2, figsize=(12, 12)) +fig, axes = plt.subplots(2, 4, figsize=(12, 12)) axes = axes.flatten() for i, image in enumerate(images): @@ -101,126 +116,18 @@ plt.tight_layout() plt.show() ``` - - - -For image-to-image, pass a list of input images and prompts to the pipeline. - -```py -import torch -from diffusers.utils import load_image -from diffusers import DiffusionPipeline - -pipeline = DiffusionPipeline.from_pretrained( - "stabilityai/stable-diffusion-xl-base-1.0", - torch_dtype=torch.float16 -).to("cuda") - -input_images = [ - load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"), - load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png"), - load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/detail-prompt.png") -] - -prompts = [ - "cinematic photo of a beautiful sunset over mountains, 35mm photograph, film, professional, 4k, highly detailed", - "cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain", - "pixel-art a cozy coffee shop interior, low-res, blocky, pixel art style, 8-bit graphics" -] - -images = pipeline( - prompt=prompts, - image=input_images, - guidance_scale=8.0, - strength=0.5 -).images - -fig, axes = plt.subplots(2, 2, figsize=(12, 12)) -axes = axes.flatten() - -for i, image in enumerate(images): - axes[i].imshow(image) - axes[i].set_title(f"Image {i+1}") - axes[i].axis('off') - -plt.tight_layout() -plt.show() -``` - -To generate multiple variations of one prompt, use the `num_images_per_prompt` argument. - -```py -import torch -import matplotlib.pyplot as plt -from diffusers.utils import load_image -from diffusers import DiffusionPipeline - -pipeline = DiffusionPipeline.from_pretrained( - "stabilityai/stable-diffusion-xl-base-1.0", - torch_dtype=torch.float16 -).to("cuda") - -input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/detail-prompt.png") - -images = pipeline( - prompt="pixel-art a cozy coffee shop interior, low-res, blocky, pixel art style, 8-bit graphics", - image=input_image, - num_images_per_prompt=4 -).images - -fig, axes = plt.subplots(2, 2, figsize=(12, 12)) -axes = axes.flatten() - -for i, image in enumerate(images): - axes[i].imshow(image) - axes[i].set_title(f"Image {i+1}") - axes[i].axis('off') - -plt.tight_layout() -plt.show() -``` - -Combine both approaches to generate different variations of different prompts. - -```py -input_images = [ - load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png"), - load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/detail-prompt.png") -] - -prompts = [ - "cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain", - "pixel-art a cozy coffee shop interior, low-res, blocky, pixel art style, 8-bit graphics" -] - -images = pipeline( - prompt=prompts, - image=input_images, - num_images_per_prompt=2, -).images - -fig, axes = plt.subplots(2, 2, figsize=(12, 12)) -axes = axes.flatten() - -for i, image in enumerate(images): - axes[i].imshow(image) - axes[i].set_title(f"Image {i+1}") - axes[i].axis('off') - -plt.tight_layout() -plt.show() -``` - - - +
+ +
## Deterministic generation Enable reproducible batch generation by passing a list of [Generator’s](https://pytorch.org/docs/stable/generated/torch.Generator.html) to the pipeline and tie each `Generator` to a seed to reuse it. -Use a list comprehension to iterate over the batch size specified in `range()` to create a unique `Generator` object for each image in the batch. +> [!TIP] +> Refer to the [Reproducibility](./reusing_seeds) docs to learn more about deterministic algorithms and the `Generator` object. -Don't multiply the `Generator` by the batch size because that only creates one `Generator` object that is used sequentially for each image in the batch. +Use a list comprehension to iterate over the batch size specified in `range()` to create a unique `Generator` object for each image in the batch. Don't multiply the `Generator` by the batch size because that only creates one `Generator` object that is used sequentially for each image in the batch. ```py generator = [torch.Generator(device="cuda").manual_seed(0)] * 3 @@ -234,14 +141,16 @@ from diffusers import DiffusionPipeline pipeline = DiffusionPipeline.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", - torch_dtype=torch.float16 -).to("cuda") + torch_dtype=torch.float16, + device_map="cuda" +) generator = [torch.Generator(device="cuda").manual_seed(i) for i in range(3)] prompts = [ - "cinematic photo of A beautiful sunset over mountains, 35mm photograph, film, professional, 4k, highly detailed", - "cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain", - "pixel-art a cozy coffee shop interior, low-res, blocky, pixel art style, 8-bit graphics" + "Cinematic shot of a cozy coffee shop interior, warm pastel light streaming through a window where a cat rests. Shallow depth of field, glowing cups in soft focus, dreamy lofi-inspired mood, nostalgic tones, framed like a quiet film scene.", + "Polaroid-style photograph of a cozy coffee shop interior, bathed in warm pastel light. A cat sits on the windowsill near steaming mugs. Soft, slightly faded tones and dreamy blur evoke nostalgia, a lofi mood, and the intimate, imperfect charm of instant film.", + "Soft watercolor illustration of a cozy coffee shop interior, pastel washes of color filling the space. A cat rests peacefully on the windowsill as warm light glows through. Gentle brushstrokes create a dreamy, lofi-inspired atmosphere with whimsical textures and nostalgic calm.", + "Isometric pixel-art illustration of a cozy coffee shop interior in detailed 8-bit style. Warm pastel light fills the space as a cat rests on the windowsill. Blocky furniture and tiny mugs add charm, low-res retro graphics enhance the nostalgic, lofi-inspired game aesthetic." ] images = pipeline( @@ -261,4 +170,4 @@ plt.tight_layout() plt.show() ``` -You can use this to iteratively select an image associated with a seed and then improve on it by crafting a more detailed prompt. \ No newline at end of file +You can use this to select an image associated with a seed and iteratively improve on it by crafting a more detailed prompt. \ No newline at end of file diff --git a/docs/source/en/using-diffusers/controlling_generation.md b/docs/source/en/using-diffusers/controlling_generation.md index aed3a8b729..b7b0ea4919 100644 --- a/docs/source/en/using-diffusers/controlling_generation.md +++ b/docs/source/en/using-diffusers/controlling_generation.md @@ -70,32 +70,6 @@ For convenience, we provide a table to denote which methods are inference-only a [InstructPix2Pix](../api/pipelines/pix2pix) is fine-tuned from Stable Diffusion to support editing input images. It takes as inputs an image and a prompt describing an edit, and it outputs the edited image. InstructPix2Pix has been explicitly trained to work well with [InstructGPT](https://openai.com/blog/instruction-following/)-like prompts. -## Pix2Pix Zero - -[Paper](https://huggingface.co/papers/2302.03027) - -[Pix2Pix Zero](../api/pipelines/pix2pix_zero) allows modifying an image so that one concept or subject is translated to another one while preserving general image semantics. - -The denoising process is guided from one conceptual embedding towards another conceptual embedding. The intermediate latents are optimized during the denoising process to push the attention maps towards reference attention maps. The reference attention maps are from the denoising process of the input image and are used to encourage semantic preservation. - -Pix2Pix Zero can be used both to edit synthetic images as well as real images. - -- To edit synthetic images, one first generates an image given a caption. - Next, we generate image captions for the concept that shall be edited and for the new target concept. We can use a model like [Flan-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5) for this purpose. Then, "mean" prompt embeddings for both the source and target concepts are created via the text encoder. Finally, the pix2pix-zero algorithm is used to edit the synthetic image. -- To edit a real image, one first generates an image caption using a model like [BLIP](https://huggingface.co/docs/transformers/model_doc/blip). Then one applies DDIM inversion on the prompt and image to generate "inverse" latents. Similar to before, "mean" prompt embeddings for both source and target concepts are created and finally the pix2pix-zero algorithm in combination with the "inverse" latents is used to edit the image. - -> [!TIP] -> Pix2Pix Zero is the first model that allows "zero-shot" image editing. This means that the model -> can edit an image in less than a minute on a consumer GPU as shown [here](../api/pipelines/pix2pix_zero#usage-example). - -As mentioned above, Pix2Pix Zero includes optimizing the latents (and not any of the UNet, VAE, or the text encoder) to steer the generation toward a specific concept. This means that the overall -pipeline might require more memory than a standard [StableDiffusionPipeline](../api/pipelines/stable_diffusion/text2img). - -> [!TIP] -> An important distinction between methods like InstructPix2Pix and Pix2Pix Zero is that the former -> involves fine-tuning the pre-trained weights while the latter does not. This means that you can -> apply Pix2Pix Zero to any of the available Stable Diffusion models. - ## Attend and Excite [Paper](https://huggingface.co/papers/2301.13826) @@ -178,14 +152,6 @@ multi-concept training by design. Like DreamBooth and Textual Inversion, Custom teach a pre-trained text-to-image diffusion model about new concepts to generate outputs involving the concept(s) of interest. -## Model Editing - -[Paper](https://huggingface.co/papers/2303.08084) - -The [text-to-image model editing pipeline](../api/pipelines/model_editing) helps you mitigate some of the incorrect implicit assumptions a pre-trained text-to-image -diffusion model might make about the subjects present in the input prompt. For example, if you prompt Stable Diffusion to generate images for "A pack of roses", the roses in the generated images -are more likely to be red. This pipeline helps you change that assumption. - ## DiffEdit [Paper](https://huggingface.co/papers/2210.11427) diff --git a/docs/source/en/using-diffusers/inference_with_lcm.md b/docs/source/en/using-diffusers/inference_with_lcm.md index d0a47449ad..cde4168d38 100644 --- a/docs/source/en/using-diffusers/inference_with_lcm.md +++ b/docs/source/en/using-diffusers/inference_with_lcm.md @@ -257,7 +257,7 @@ LCMs are compatible with adapters like LoRA, ControlNet, T2I-Adapter, and Animat ### LoRA -[LoRA](../using-diffusers/loading_adapters#lora) adapters can be rapidly finetuned to learn a new style from just a few images and plugged into a pretrained model to generate images in that style. +[LoRA](../tutorials/using_peft_for_inference) adapters can be rapidly finetuned to learn a new style from just a few images and plugged into a pretrained model to generate images in that style. diff --git a/docs/source/en/using-diffusers/inference_with_tcd_lora.md b/docs/source/en/using-diffusers/inference_with_tcd_lora.md index a4de12b5e7..2aaf9c8aa8 100644 --- a/docs/source/en/using-diffusers/inference_with_tcd_lora.md +++ b/docs/source/en/using-diffusers/inference_with_tcd_lora.md @@ -18,7 +18,7 @@ Trajectory Consistency Distillation (TCD) enables a model to generate higher qua The major advantages of TCD are: -- Better than Teacher: TCD demonstrates superior generative quality at both small and large inference steps and exceeds the performance of [DPM-Solver++(2S)](../../api/schedulers/multistep_dpm_solver) with Stable Diffusion XL (SDXL). There is no additional discriminator or LPIPS supervision included during TCD training. +- Better than Teacher: TCD demonstrates superior generative quality at both small and large inference steps and exceeds the performance of [DPM-Solver++(2S)](../api/schedulers/multistep_dpm_solver) with Stable Diffusion XL (SDXL). There is no additional discriminator or LPIPS supervision included during TCD training. - Flexible Inference Steps: The inference steps for TCD sampling can be freely adjusted without adversely affecting the image quality. @@ -166,7 +166,7 @@ image = pipe( TCD-LoRA also supports other LoRAs trained on different styles. For example, let's load the [TheLastBen/Papercut_SDXL](https://huggingface.co/TheLastBen/Papercut_SDXL) LoRA and fuse it with the TCD-LoRA with the [`~loaders.UNet2DConditionLoadersMixin.set_adapters`] method. > [!TIP] -> Check out the [Merge LoRAs](merge_loras) guide to learn more about efficient merging methods. +> Check out the [Merge LoRAs](../tutorials/using_peft_for_inference#merge) guide to learn more about efficient merging methods. ```python import torch diff --git a/docs/source/en/using-diffusers/sdxl.md b/docs/source/en/using-diffusers/sdxl.md index 79625e0c4a..275394a03c 100644 --- a/docs/source/en/using-diffusers/sdxl.md +++ b/docs/source/en/using-diffusers/sdxl.md @@ -280,7 +280,7 @@ refiner = DiffusionPipeline.from_pretrained( ``` > [!WARNING] -> You can use SDXL refiner with a different base model. For example, you can use the [Hunyuan-DiT](../../api/pipelines/hunyuandit) or [PixArt-Sigma](../../api/pipelines/pixart_sigma) pipelines to generate images with better prompt adherence. Once you have generated an image, you can pass it to the SDXL refiner model to enhance final generation quality. +> You can use SDXL refiner with a different base model. For example, you can use the [Hunyuan-DiT](../api/pipelines/hunyuandit) or [PixArt-Sigma](../api/pipelines/pixart_sigma) pipelines to generate images with better prompt adherence. Once you have generated an image, you can pass it to the SDXL refiner model to enhance final generation quality. Generate an image from the base model, and set the model output to **latent** space: diff --git a/docs/source/en/using-diffusers/weighted_prompts.md b/docs/source/en/using-diffusers/weighted_prompts.md index b45568ac4d..f89ebfe4a2 100644 --- a/docs/source/en/using-diffusers/weighted_prompts.md +++ b/docs/source/en/using-diffusers/weighted_prompts.md @@ -10,423 +10,96 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# Prompt techniques - [[open-in-colab]] -Prompts are important because they describe what you want a diffusion model to generate. The best prompts are detailed, specific, and well-structured to help the model realize your vision. But crafting a great prompt takes time and effort and sometimes it may not be enough because language and words can be imprecise. This is where you need to boost your prompt with other techniques, such as prompt enhancing and prompt weighting, to get the results you want. +# Prompting -This guide will show you how you can use these prompt techniques to generate high-quality images with lower effort and adjust the weight of certain keywords in a prompt. +Prompts describes what a model should generate. Good prompts are detailed, specific, and structured and they generate better images and videos. -## Prompt engineering +This guide shows you how to write effective prompts and introduces techniques that make them stronger. -> [!TIP] -> This is not an exhaustive guide on prompt engineering, but it will help you understand the necessary parts of a good prompt. We encourage you to continue experimenting with different prompts and combine them in new ways to see what works best. As you write more prompts, you'll develop an intuition for what works and what doesn't! +## Writing good prompts -New diffusion models do a pretty good job of generating high-quality images from a basic prompt, but it is still important to create a well-written prompt to get the best results. Here are a few tips for writing a good prompt: +Every effective prompt needs three core elements. -1. What is the image *medium*? Is it a photo, a painting, a 3D illustration, or something else? -2. What is the image *subject*? Is it a person, animal, object, or scene? -3. What *details* would you like to see in the image? This is where you can get really creative and have a lot of fun experimenting with different words to bring your image to life. For example, what is the lighting like? What is the vibe and aesthetic? What kind of art or illustration style are you looking for? The more specific and precise words you use, the better the model will understand what you want to generate. +1. Subject - what you want to generate. Start your prompt here. +2. Style - the medium or aesthetic. How should it look? +3. Context - details about actions, setting, and mood. + +Use these elements as a structured narrative, not a keyword list. Modern models understand language better than keyword matching. Start simple, then add details. + +Context is especially important for creating better prompts. Try adding lighting, artistic details, and mood.
-
- -
"A photo of a banana-shaped couch in a living room"
+
+ +
A cute cat lounges on a leaf in a pool during a peaceful summer afternoon, in lofi art style, illustration.
-
- -
"A vibrant yellow banana-shaped couch sits in a cozy living room, its curve cradling a pile of colorful cushions. on the wooden floor, a patterned rug adds a touch of eclectic charm, and a potted plant sits in the corner, reaching towards the sunlight filtering through the windows"
+
+ +
A cute cat lounges on a floating leaf in a sparkling pool during a peaceful summer afternoon. Clear reflections ripple across the water, with sunlight casting soft, smooth highlights. The illustration is detailed and polished, with elegant lines and harmonious colors, evoking a relaxing, serene, and whimsical lofi mood, anime-inspired and visually comforting.
-## Prompt enhancing with GPT2 - -Prompt enhancing is a technique for quickly improving prompt quality without spending too much effort constructing one. It uses a model like GPT2 pretrained on Stable Diffusion text prompts to automatically enrich a prompt with additional important keywords to generate high-quality images. - -The technique works by curating a list of specific keywords and forcing the model to generate those words to enhance the original prompt. This way, your prompt can be "a cat" and GPT2 can enhance the prompt to "cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain quality sharp focus beautiful detailed intricate stunning amazing epic". +Be specific and add context. Use photography terms like lens type, focal length, camera angles, and depth of field. > [!TIP] -> You should also use a [*offset noise*](https://www.crosslabs.org//blog/diffusion-with-offset-noise) LoRA to improve the contrast in bright and dark images and create better lighting overall. This [LoRA](https://hf.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_offset_example-lora_1.0.safetensors) is available from [stabilityai/stable-diffusion-xl-base-1.0](https://hf.co/stabilityai/stable-diffusion-xl-base-1.0). - -Start by defining certain styles and a list of words (you can check out a more comprehensive list of [words](https://hf.co/LykosAI/GPT-Prompt-Expansion-Fooocus-v2/blob/main/positive.txt) and [styles](https://github.com/lllyasviel/Fooocus/tree/main/sdxl_styles) used by Fooocus) to enhance a prompt with. - -```py -import torch -from transformers import GenerationConfig, GPT2LMHeadModel, GPT2Tokenizer, LogitsProcessor, LogitsProcessorList -from diffusers import StableDiffusionXLPipeline - -styles = { - "cinematic": "cinematic film still of {prompt}, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain", - "anime": "anime artwork of {prompt}, anime style, key visual, vibrant, studio anime, highly detailed", - "photographic": "cinematic photo of {prompt}, 35mm photograph, film, professional, 4k, highly detailed", - "comic": "comic of {prompt}, graphic illustration, comic art, graphic novel art, vibrant, highly detailed", - "lineart": "line art drawing {prompt}, professional, sleek, modern, minimalist, graphic, line art, vector graphics", - "pixelart": " pixel-art {prompt}, low-res, blocky, pixel art style, 8-bit graphics", -} - -words = [ - "aesthetic", "astonishing", "beautiful", "breathtaking", "composition", "contrasted", "epic", "moody", "enhanced", - "exceptional", "fascinating", "flawless", "glamorous", "glorious", "illumination", "impressive", "improved", - "inspirational", "magnificent", "majestic", "hyperrealistic", "smooth", "sharp", "focus", "stunning", "detailed", - "intricate", "dramatic", "high", "quality", "perfect", "light", "ultra", "highly", "radiant", "satisfying", - "soothing", "sophisticated", "stylish", "sublime", "terrific", "touching", "timeless", "wonderful", "unbelievable", - "elegant", "awesome", "amazing", "dynamic", "trendy", -] -``` - -You may have noticed in the `words` list, there are certain words that can be paired together to create something more meaningful. For example, the words "high" and "quality" can be combined to create "high quality". Let's pair these words together and remove the words that can't be paired. - -```py -word_pairs = ["highly detailed", "high quality", "enhanced quality", "perfect composition", "dynamic light"] - -def find_and_order_pairs(s, pairs): - words = s.split() - found_pairs = [] - for pair in pairs: - pair_words = pair.split() - if pair_words[0] in words and pair_words[1] in words: - found_pairs.append(pair) - words.remove(pair_words[0]) - words.remove(pair_words[1]) - - for word in words[:]: - for pair in pairs: - if word in pair.split(): - words.remove(word) - break - ordered_pairs = ", ".join(found_pairs) - remaining_s = ", ".join(words) - return ordered_pairs, remaining_s -``` - -Next, implement a custom [`~transformers.LogitsProcessor`] class that assigns tokens in the `words` list a value of 0 and assigns tokens not in the `words` list a negative value so they aren't picked during generation. This way, generation is biased towards words in the `words` list. After a word from the list is used, it is also assigned a negative value so it isn't picked again. - -```py -class CustomLogitsProcessor(LogitsProcessor): - def __init__(self, bias): - super().__init__() - self.bias = bias - - def __call__(self, input_ids, scores): - if len(input_ids.shape) == 2: - last_token_id = input_ids[0, -1] - self.bias[last_token_id] = -1e10 - return scores + self.bias - -word_ids = [tokenizer.encode(word, add_prefix_space=True)[0] for word in words] -bias = torch.full((tokenizer.vocab_size,), -float("Inf")).to("cuda") -bias[word_ids] = 0 -processor = CustomLogitsProcessor(bias) -processor_list = LogitsProcessorList([processor]) -``` - -Combine the prompt and the `cinematic` style prompt defined in the `styles` dictionary earlier. - -```py -prompt = "a cat basking in the sun on a roof in Turkey" -style = "cinematic" - -prompt = styles[style].format(prompt=prompt) -prompt -"cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain" -``` - -Load a GPT2 tokenizer and model from the [Gustavosta/MagicPrompt-Stable-Diffusion](https://huggingface.co/Gustavosta/MagicPrompt-Stable-Diffusion) checkpoint (this specific checkpoint is trained to generate prompts) to enhance the prompt. - -```py -tokenizer = GPT2Tokenizer.from_pretrained("Gustavosta/MagicPrompt-Stable-Diffusion") -model = GPT2LMHeadModel.from_pretrained("Gustavosta/MagicPrompt-Stable-Diffusion", torch_dtype=torch.float16).to( - "cuda" -) -model.eval() - -inputs = tokenizer(prompt, return_tensors="pt").to("cuda") -token_count = inputs["input_ids"].shape[1] -max_new_tokens = 50 - token_count - -generation_config = GenerationConfig( - penalty_alpha=0.7, - top_k=50, - eos_token_id=model.config.eos_token_id, - pad_token_id=model.config.eos_token_id, - pad_token=model.config.pad_token_id, - do_sample=True, -) - -with torch.no_grad(): - generated_ids = model.generate( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - max_new_tokens=max_new_tokens, - generation_config=generation_config, - logits_processor=proccesor_list, - ) -``` - -Then you can combine the input prompt and the generated prompt. Feel free to take a look at what the generated prompt (`generated_part`) is, the word pairs that were found (`pairs`), and the remaining words (`words`). This is all packed together in the `enhanced_prompt`. - -```py -output_tokens = [tokenizer.decode(generated_id, skip_special_tokens=True) for generated_id in generated_ids] -input_part, generated_part = output_tokens[0][: len(prompt)], output_tokens[0][len(prompt) :] -pairs, words = find_and_order_pairs(generated_part, word_pairs) -formatted_generated_part = pairs + ", " + words -enhanced_prompt = input_part + ", " + formatted_generated_part -enhanced_prompt -["cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain quality sharp focus beautiful detailed intricate stunning amazing epic"] -``` - -Finally, load a pipeline and the offset noise LoRA with a *low weight* to generate an image with the enhanced prompt. - -```py -pipeline = StableDiffusionXLPipeline.from_pretrained( - "RunDiffusion/Juggernaut-XL-v9", torch_dtype=torch.float16, variant="fp16" -).to("cuda") - -pipeline.load_lora_weights( - "stabilityai/stable-diffusion-xl-base-1.0", - weight_name="sd_xl_offset_example-lora_1.0.safetensors", - adapter_name="offset", -) -pipeline.set_adapters(["offset"], adapter_weights=[0.2]) - -image = pipeline( - enhanced_prompt, - width=1152, - height=896, - guidance_scale=7.5, - num_inference_steps=25, -).images[0] -image -``` - -
-
- -
"a cat basking in the sun on a roof in Turkey"
-
-
- -
"cinematic film still of a cat basking in the sun on a roof in Turkey, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain"
-
-
+> Try a [prompt enhancer](https://huggingface.co/models?sort=downloads&search=prompt+enhancer) to help improve your prompt structure. ## Prompt weighting -Prompt weighting provides a way to emphasize or de-emphasize certain parts of a prompt, allowing for more control over the generated image. A prompt can include several concepts, which gets turned into contextualized text embeddings. The embeddings are used by the model to condition its cross-attention layers to generate an image (read the Stable Diffusion [blog post](https://huggingface.co/blog/stable_diffusion) to learn more about how it works). +Prompt weighting makes some words stronger and others weaker. It scales attention scores so you control how much influence each concept has. -Prompt weighting works by increasing or decreasing the scale of the text embedding vector that corresponds to its concept in the prompt because you may not necessarily want the model to focus on all concepts equally. The easiest way to prepare the prompt embeddings is to use [Stable Diffusion Long Prompt Weighted Embedding](https://github.com/xhinker/sd_embed) (sd_embed). Once you have the prompt-weighted embeddings, you can pass them to any pipeline that has a [prompt_embeds](https://huggingface.co/docs/diffusers/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.__call__.prompt_embeds) (and optionally [negative_prompt_embeds](https://huggingface.co/docs/diffusers/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.__call__.negative_prompt_embeds)) parameter, such as [`StableDiffusionPipeline`], [`StableDiffusionControlNetPipeline`], and [`StableDiffusionXLPipeline`]. +Diffusers handles this through `prompt_embeds` and `pooled_prompt_embeds` arguments which take scaled text embedding vectors. Use the [sd_embed](https://github.com/xhinker/sd_embed) library to generate these embeddings. It also supports longer prompts. -> [!TIP] -> If your favorite pipeline doesn't have a `prompt_embeds` parameter, please open an [issue](https://github.com/huggingface/diffusers/issues/new/choose) so we can add it! - -This guide will show you how to weight your prompts with sd_embed. - -Before you begin, make sure you have the latest version of sd_embed installed: - -```bash -pip install git+https://github.com/xhinker/sd_embed.git@main -``` - -For this example, let's use [`StableDiffusionXLPipeline`]. +> [!NOTE] +> The sd_embed library only supports Stable Diffusion, Stable Diffusion XL, Stable Diffusion 3, Stable Cascade, and Flux. Prompt weighting doesn't necessarily help for newer models like Flux which already has very good prompt adherence. ```py -from diffusers import StableDiffusionXLPipeline, UniPCMultistepScheduler -import torch - -pipe = StableDiffusionXLPipeline.from_pretrained("Lykon/dreamshaper-xl-1-0", torch_dtype=torch.float16) -pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) -pipe.to("cuda") +!uv pip install git+https://github.com/xhinker/sd_embed.git@main ``` -To upweight or downweight a concept, surround the text with parentheses. More parentheses applies a heavier weight on the text. You can also append a numerical multiplier to the text to indicate how much you want to increase or decrease its weights by. +Format weighted text with numerical multipliers or parentheses. More parentheses mean stronger weighting. | format | multiplier | |---|---| -| `(hippo)` | increase by 1.1x | -| `((hippo))` | increase by 1.21x | -| `(hippo:1.5)` | increase by 1.5x | -| `(hippo:0.5)` | decrease by 4x | +| `(cat)` | increase by 1.1x | +| `((cat))` | increase by 1.21x | +| `(cat:1.5)` | increase by 1.5x | +| `(cat:0.5)` | decrease by 4x | -Create a prompt and use a combination of parentheses and numerical multipliers to upweight various text. +Create a weighted prompt and pass it to [get_weighted_text_embeddings_sdxl](https://github.com/xhinker/sd_embed/blob/4a47f71150a22942fa606fb741a1c971d95ba56f/src/sd_embed/embedding_funcs.py#L405) to generate embeddings. + +> [!TIP] +> You could also pass negative prompts to `negative_prompt_embeds` and `negative_pooled_prompt_embeds`. ```py +import torch +from diffusers import DiffusionPipeline from sd_embed.embedding_funcs import get_weighted_text_embeddings_sdxl -prompt = """A whimsical and creative image depicting a hybrid creature that is a mix of a waffle and a hippopotamus. -This imaginative creature features the distinctive, bulky body of a hippo, -but with a texture and appearance resembling a golden-brown, crispy waffle. -The creature might have elements like waffle squares across its skin and a syrup-like sheen. -It's set in a surreal environment that playfully combines a natural water habitat of a hippo with elements of a breakfast table setting, -possibly including oversized utensils or plates in the background. -The image should evoke a sense of playful absurdity and culinary fantasy. -""" - -neg_prompt = """\ -skin spots,acnes,skin blemishes,age spot,(ugly:1.2),(duplicate:1.2),(morbid:1.21),(mutilated:1.2),\ -(tranny:1.2),mutated hands,(poorly drawn hands:1.5),blurry,(bad anatomy:1.2),(bad proportions:1.3),\ -extra limbs,(disfigured:1.2),(missing arms:1.2),(extra legs:1.2),(fused fingers:1.5),\ -(too many fingers:1.5),(unclear eyes:1.2),lowers,bad hands,missing fingers,extra digit,\ -bad hands,missing fingers,(extra arms and legs),(worst quality:2),(low quality:2),\ -(normal quality:2),lowres,((monochrome)),((grayscale)) -""" -``` - -Use the `get_weighted_text_embeddings_sdxl` function to generate the prompt embeddings and the negative prompt embeddings. It'll also generated the pooled and negative pooled prompt embeddings since you're using the SDXL model. - -> [!TIP] -> You can safely ignore the error message below about the token index length exceeding the models maximum sequence length. All your tokens will be used in the embedding process. -> -> ``` -> Token indices sequence length is longer than the specified maximum sequence length for this model -> ``` - -```py -( - prompt_embeds, - prompt_neg_embeds, - pooled_prompt_embeds, - negative_pooled_prompt_embeds -) = get_weighted_text_embeddings_sdxl( - pipe, - prompt=prompt, - neg_prompt=neg_prompt +pipeline = DiffusionPipeline.from_pretrained( + "Lykon/dreamshaper-xl-1-0", torch_dtype=torch.bfloat16, device_map="cuda" ) -image = pipe( - prompt_embeds=prompt_embeds, - negative_prompt_embeds=prompt_neg_embeds, - pooled_prompt_embeds=pooled_prompt_embeds, - negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, - num_inference_steps=30, - height=1024, - width=1024 + 512, - guidance_scale=4.0, - generator=torch.Generator("cuda").manual_seed(2) -).images[0] -image +prompt = """ +A (cute cat:1.4) lounges on a (floating leaf:1.2) in a (sparkling pool:1.1) during a peaceful summer afternoon. +Gentle ripples reflect pastel skies, while (sunlight:1.1) casts soft highlights. The illustration is smooth and polished +with elegant, sketchy lines and subtle gradients, evoking a ((whimsical, nostalgic, dreamy lofi atmosphere:2.0)), +(anime-inspired:1.6), calming, comforting, and visually serene. +""" + +prompt_embeds, _, pooled_prompt_embeds, *_ = get_weighted_text_embeddings_sdxl(pipeline, prompt=prompt) +``` + +Pass the embeddings to `prompt_embeds` and `pooled_prompt_embeds` to generate your image. + +```py +image = pipeline(prompt_embeds=prompt_embeds, pooled_prompt_embeds=pooled_prompt_embeds).images[0] ```
- +
-> [!TIP] -> Refer to the [sd_embed](https://github.com/xhinker/sd_embed) repository for additional details about long prompt weighting for FLUX.1, Stable Cascade, and Stable Diffusion 1.5. - -### Textual inversion - -[Textual inversion](../training/text_inversion) is a technique for learning a specific concept from some images which you can use to generate new images conditioned on that concept. - -Create a pipeline and use the [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] function to load the textual inversion embeddings (feel free to browse the [Stable Diffusion Conceptualizer](https://huggingface.co/spaces/sd-concepts-library/stable-diffusion-conceptualizer) for 100+ trained concepts): - -```py -import torch -from diffusers import StableDiffusionPipeline - -pipe = StableDiffusionPipeline.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", - torch_dtype=torch.float16, -).to("cuda") -pipe.load_textual_inversion("sd-concepts-library/midjourney-style") -``` - -Add the `` text to the prompt to trigger the textual inversion. - -```py -from sd_embed.embedding_funcs import get_weighted_text_embeddings_sd15 - -prompt = """ A whimsical and creative image depicting a hybrid creature that is a mix of a waffle and a hippopotamus. -This imaginative creature features the distinctive, bulky body of a hippo, -but with a texture and appearance resembling a golden-brown, crispy waffle. -The creature might have elements like waffle squares across its skin and a syrup-like sheen. -It's set in a surreal environment that playfully combines a natural water habitat of a hippo with elements of a breakfast table setting, -possibly including oversized utensils or plates in the background. -The image should evoke a sense of playful absurdity and culinary fantasy. -""" - -neg_prompt = """\ -skin spots,acnes,skin blemishes,age spot,(ugly:1.2),(duplicate:1.2),(morbid:1.21),(mutilated:1.2),\ -(tranny:1.2),mutated hands,(poorly drawn hands:1.5),blurry,(bad anatomy:1.2),(bad proportions:1.3),\ -extra limbs,(disfigured:1.2),(missing arms:1.2),(extra legs:1.2),(fused fingers:1.5),\ -(too many fingers:1.5),(unclear eyes:1.2),lowers,bad hands,missing fingers,extra digit,\ -bad hands,missing fingers,(extra arms and legs),(worst quality:2),(low quality:2),\ -(normal quality:2),lowres,((monochrome)),((grayscale)) -""" -``` - -Use the `get_weighted_text_embeddings_sd15` function to generate the prompt embeddings and the negative prompt embeddings. - -```py -( - prompt_embeds, - prompt_neg_embeds, -) = get_weighted_text_embeddings_sd15( - pipe, - prompt=prompt, - neg_prompt=neg_prompt -) - -image = pipe( - prompt_embeds=prompt_embeds, - negative_prompt_embeds=prompt_neg_embeds, - height=768, - width=896, - guidance_scale=4.0, - generator=torch.Generator("cuda").manual_seed(2) -).images[0] -image -``` - -
- -
- -### DreamBooth - -[DreamBooth](../training/dreambooth) is a technique for generating contextualized images of a subject given just a few images of the subject to train on. It is similar to textual inversion, but DreamBooth trains the full model whereas textual inversion only fine-tunes the text embeddings. This means you should use [`~DiffusionPipeline.from_pretrained`] to load the DreamBooth model (feel free to browse the [Stable Diffusion Dreambooth Concepts Library](https://huggingface.co/sd-dreambooth-library) for 100+ trained models): - -```py -import torch -from diffusers import DiffusionPipeline, UniPCMultistepScheduler - -pipe = DiffusionPipeline.from_pretrained("sd-dreambooth-library/dndcoverart-v1", torch_dtype=torch.float16).to("cuda") -pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) -``` - -Depending on the model you use, you'll need to incorporate the model's unique identifier into your prompt. For example, the `dndcoverart-v1` model uses the identifier `dndcoverart`: - -```py -from sd_embed.embedding_funcs import get_weighted_text_embeddings_sd15 - -prompt = """dndcoverart of A whimsical and creative image depicting a hybrid creature that is a mix of a waffle and a hippopotamus. -This imaginative creature features the distinctive, bulky body of a hippo, -but with a texture and appearance resembling a golden-brown, crispy waffle. -The creature might have elements like waffle squares across its skin and a syrup-like sheen. -It's set in a surreal environment that playfully combines a natural water habitat of a hippo with elements of a breakfast table setting, -possibly including oversized utensils or plates in the background. -The image should evoke a sense of playful absurdity and culinary fantasy. -""" - -neg_prompt = """\ -skin spots,acnes,skin blemishes,age spot,(ugly:1.2),(duplicate:1.2),(morbid:1.21),(mutilated:1.2),\ -(tranny:1.2),mutated hands,(poorly drawn hands:1.5),blurry,(bad anatomy:1.2),(bad proportions:1.3),\ -extra limbs,(disfigured:1.2),(missing arms:1.2),(extra legs:1.2),(fused fingers:1.5),\ -(too many fingers:1.5),(unclear eyes:1.2),lowers,bad hands,missing fingers,extra digit,\ -bad hands,missing fingers,(extra arms and legs),(worst quality:2),(low quality:2),\ -(normal quality:2),lowres,((monochrome)),((grayscale)) -""" - -( - prompt_embeds - , prompt_neg_embeds -) = get_weighted_text_embeddings_sd15( - pipe - , prompt = prompt - , neg_prompt = neg_prompt -) -``` - -
- -
+Prompt weighting works with [Textual inversion](./textual_inversion_inference) and [DreamBooth](./dreambooth) adapters too. \ No newline at end of file diff --git a/docs/source/en/using-diffusers/write_own_pipeline.md b/docs/source/en/using-diffusers/write_own_pipeline.md index 930b0fe21f..e34727b5da 100644 --- a/docs/source/en/using-diffusers/write_own_pipeline.md +++ b/docs/source/en/using-diffusers/write_own_pipeline.md @@ -280,5 +280,5 @@ This is really what 🧨 Diffusers is designed for: to make it intuitive and eas For your next steps, feel free to: -* Learn how to [build and contribute a pipeline](../using-diffusers/contribute_pipeline) to 🧨 Diffusers. We can't wait and see what you'll come up with! +* Learn how to [build and contribute a pipeline](../conceptual/contribution) to 🧨 Diffusers. We can't wait and see what you'll come up with! * Explore [existing pipelines](../api/pipelines/overview) in the library, and see if you can deconstruct and build a pipeline from scratch using the models and schedulers separately. diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py b/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py index 7b0f9889a5..dc5e775f67 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py @@ -18,7 +18,6 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...utils import logging diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py b/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py index 87ac406592..9872cf0968 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py @@ -23,7 +23,6 @@ from typing import List, Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import FromOriginalModelMixin diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_wan.py b/src/diffusers/models/autoencoders/autoencoder_kl_wan.py index e6e58c1cce..f95c4cf374 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_wan.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_wan.py @@ -17,7 +17,6 @@ from typing import List, Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import FromOriginalModelMixin diff --git a/src/diffusers/models/controlnets/controlnet_xs.py b/src/diffusers/models/controlnets/controlnet_xs.py index bcb4e25986..f5c69b9a46 100644 --- a/src/diffusers/models/controlnets/controlnet_xs.py +++ b/src/diffusers/models/controlnets/controlnet_xs.py @@ -16,7 +16,6 @@ from math import gcd from typing import Any, Dict, List, Optional, Tuple, Union import torch -import torch.utils.checkpoint from torch import Tensor, nn from ...configuration_utils import ConfigMixin, register_to_config diff --git a/src/diffusers/models/transformers/stable_audio_transformer.py b/src/diffusers/models/transformers/stable_audio_transformer.py index 969e6db122..ac9b3fca41 100644 --- a/src/diffusers/models/transformers/stable_audio_transformer.py +++ b/src/diffusers/models/transformers/stable_audio_transformer.py @@ -18,7 +18,6 @@ from typing import Dict, Optional, Union import numpy as np import torch import torch.nn as nn -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...utils import logging diff --git a/src/diffusers/models/unets/unet_2d_condition.py b/src/diffusers/models/unets/unet_2d_condition.py index 33bda8cb1e..f04d3dfa01 100644 --- a/src/diffusers/models/unets/unet_2d_condition.py +++ b/src/diffusers/models/unets/unet_2d_condition.py @@ -16,7 +16,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union import torch import torch.nn as nn -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import PeftAdapterMixin, UNet2DConditionLoadersMixin diff --git a/src/diffusers/models/unets/unet_3d_condition.py b/src/diffusers/models/unets/unet_3d_condition.py index b5151f3c9a..6a119185b8 100644 --- a/src/diffusers/models/unets/unet_3d_condition.py +++ b/src/diffusers/models/unets/unet_3d_condition.py @@ -18,7 +18,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union import torch import torch.nn as nn -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import UNet2DConditionLoadersMixin diff --git a/src/diffusers/models/unets/unet_i2vgen_xl.py b/src/diffusers/models/unets/unet_i2vgen_xl.py index 7148723a84..3dba8edca7 100644 --- a/src/diffusers/models/unets/unet_i2vgen_xl.py +++ b/src/diffusers/models/unets/unet_i2vgen_xl.py @@ -16,7 +16,6 @@ from typing import Any, Dict, Optional, Tuple, Union import torch import torch.nn as nn -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import UNet2DConditionLoadersMixin diff --git a/src/diffusers/models/unets/unet_kandinsky3.py b/src/diffusers/models/unets/unet_kandinsky3.py index 423669a22f..27241ce2e6 100644 --- a/src/diffusers/models/unets/unet_kandinsky3.py +++ b/src/diffusers/models/unets/unet_kandinsky3.py @@ -16,7 +16,6 @@ from dataclasses import dataclass from typing import Dict, Tuple, Union import torch -import torch.utils.checkpoint from torch import nn from ...configuration_utils import ConfigMixin, register_to_config diff --git a/src/diffusers/models/unets/unet_motion_model.py b/src/diffusers/models/unets/unet_motion_model.py index 26616e53bd..18d5eb917f 100644 --- a/src/diffusers/models/unets/unet_motion_model.py +++ b/src/diffusers/models/unets/unet_motion_model.py @@ -18,7 +18,6 @@ from typing import Any, Dict, Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, FrozenDict, register_to_config from ...loaders import FromOriginalModelMixin, PeftAdapterMixin, UNet2DConditionLoadersMixin diff --git a/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py b/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py index 546ae9239a..b6b40cd6e6 100644 --- a/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py +++ b/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py @@ -17,7 +17,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union import torch import torch.nn as nn -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import UNet2DConditionLoadersMixin diff --git a/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py b/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py index 928698e442..b061ac2636 100644 --- a/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py +++ b/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py @@ -14,7 +14,6 @@ from typing import Optional, Tuple, Union import torch -import torch.utils.checkpoint from torch import nn from transformers import BertTokenizer from transformers.activations import QuickGELUActivation as QuickGELU diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py index 2beb0be57b..034a022641 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py @@ -18,7 +18,6 @@ from typing import Callable, List, Optional, Union import numpy as np import PIL.Image import torch -import torch.utils.checkpoint from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection from ....image_processor import VaeImageProcessor diff --git a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py index adfd899e76..2f54f4fc98 100644 --- a/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +++ b/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py @@ -16,7 +16,6 @@ import inspect from typing import Callable, List, Optional, Union import torch -import torch.utils.checkpoint from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer from ....image_processor import VaeImageProcessor diff --git a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py index bc50835d19..f1bf4701e3 100644 --- a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +++ b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py @@ -17,7 +17,6 @@ from typing import List, Optional, Tuple, Union import torch import torch.nn as nn -import torch.utils.checkpoint from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizer from transformers.activations import ACT2FN from transformers.modeling_outputs import BaseModelOutput diff --git a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py index 273e97f1ec..631539e5c6 100644 --- a/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py +++ b/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py @@ -4,7 +4,6 @@ from typing import List, Optional, Tuple, Union import numpy as np import PIL.Image import torch -import torch.utils.checkpoint from ...models import UNet2DModel, VQModel from ...schedulers import ( diff --git a/src/diffusers/pipelines/stable_audio/modeling_stable_audio.py b/src/diffusers/pipelines/stable_audio/modeling_stable_audio.py index 89d4d2dca5..07b382dfc4 100644 --- a/src/diffusers/pipelines/stable_audio/modeling_stable_audio.py +++ b/src/diffusers/pipelines/stable_audio/modeling_stable_audio.py @@ -18,7 +18,6 @@ from typing import Optional import torch import torch.nn as nn -import torch.utils.checkpoint from ...configuration_utils import ConfigMixin, register_to_config from ...models.modeling_utils import ModelMixin diff --git a/tests/pipelines/audioldm2/test_audioldm2.py b/tests/pipelines/audioldm2/test_audioldm2.py index e4bc5cc110..14ff1272a2 100644 --- a/tests/pipelines/audioldm2/test_audioldm2.py +++ b/tests/pipelines/audioldm2/test_audioldm2.py @@ -138,10 +138,8 @@ class AudioLDM2PipelineFastTests(PipelineTesterMixin, unittest.TestCase): patch_stride=2, patch_embed_input_channels=4, ) - text_encoder_config = ClapConfig.from_text_audio_configs( - text_config=text_branch_config, - audio_config=audio_branch_config, - projection_dim=16, + text_encoder_config = ClapConfig( + text_config=text_branch_config, audio_config=audio_branch_config, projection_dim=16 ) text_encoder = ClapModel(text_encoder_config) tokenizer = RobertaTokenizer.from_pretrained("hf-internal-testing/tiny-random-roberta", model_max_length=77)