ChronoEdit

来自 NVIDIA 和多伦多大学的 ChronoEdit: Towards Temporal Reasoning for Image Editing and World Simulation，作者为 Jay Zhangjie Wu、Xuanchi Ren、Tianchang Shen、Tianshi Cao、Kai He、Yifan Lu、Ruiyuan Gao、Enze Xie、Shiyi Lan、Jose M. Alvarez、Jun Gao、Sanja Fidler、Zian Wang 和 Huan Ling。

摘要：ChronoEdit 将图像编辑重新定义为视频生成任务，利用输入图像和编辑后的图像作为起始帧和结束帧，从而借助具有时间一致性的预训练视频模型。时间推理阶段引入了推理 token，以确保编辑结果在物理上合理，并可视化编辑轨迹。

大型生成模型的近期进展极大地提升了图像编辑和上下文图像生成的能力，但在确保物理一致性方面仍存在关键差距，即编辑后的对象必须保持连贯。这一能力对于世界模拟相关任务尤为重要。本文提出了 ChronoEdit，一个将图像编辑重新定义为视频生成问题的框架。首先，ChronoEdit 将输入图像和编辑后的图像分别视为视频的首帧和末帧，从而能够利用大型预训练视频生成模型，这些模型不仅捕捉对象的外观，还通过习得的时间一致性捕捉运动和交互的隐含物理规律。其次，ChronoEdit 引入了一个时间推理阶段，在推理时显式执行编辑。在此设置下，目标帧与推理 token 联合去噪，以想象一条合理的编辑轨迹，从而将解空间约束在物理上可行的变换范围内。随后，在几个步骤后丢弃推理 token，以避免渲染完整视频带来的高昂计算成本。为了验证 ChronoEdit，我们引入了 PBench-Edit，这是一个针对需要物理一致性的上下文的新图像-提示词对基准，并证明 ChronoEdit 在视觉保真度和物理合理性方面均超越了最先进的基线方法。代码和模型的项目页面：this https URL。

ChronoEdit 流水线由 ChronoEdit 团队开发。原始代码可在 GitHub 上获取，预训练模型可在 Hugging Face 的 nvidia/ChronoEdit 集合中找到。

可用的模型/LoRA：

图像编辑

import torch
import numpy as np
from diffusers import AutoencoderKLWan, ChronoEditTransformer3DModel, ChronoEditPipeline
from diffusers.utils import export_to_video, load_image
from transformers import CLIPVisionModel
from PIL import Image

model_id = "nvidia/ChronoEdit-14B-Diffusers"
image_encoder = CLIPVisionModel.from_pretrained(model_id, subfolder="image_encoder", dtype=torch.float32)
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", dtype=torch.float32)
transformer = ChronoEditTransformer3DModel.from_pretrained(model_id, subfolder="transformer", dtype=torch.bfloat16)
pipe = ChronoEditPipeline.from_pretrained(model_id, image_encoder=image_encoder, transformer=transformer, vae=vae, dtype=torch.bfloat16)
pipe.to("cuda")  # or "mps", "xpu", "cpu"

image = load_image(
    "https://huggingface.co/spaces/nvidia/ChronoEdit/resolve/main/examples/3.png"
)
max_area = 720 * 1280
aspect_ratio = image.height / image.width
mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
print("width", width, "height", height)
image = image.resize((width, height))
prompt = (
    "The user wants to transform the image by adding a small, cute mouse sitting inside the floral teacup, enjoying a spa bath. The mouse should appear relaxed and cheerful, with a tiny white bath towel draped over its head like a turban. It should be positioned comfortably in the cup’s liquid, with gentle steam rising around it to blend with the cozy atmosphere. "
    "The mouse’s pose should be natural—perhaps sitting upright with paws resting lightly on the rim or submerged in the tea. The teacup’s floral design, gold trim, and warm lighting must remain unchanged to preserve the original aesthetic. The steam should softly swirl around the mouse, enhancing the spa-like, whimsical mood."
)

output = pipe(
    image=image,
    prompt=prompt,
    height=height,
    width=width,
    num_frames=5,
    num_inference_steps=50,
    guidance_scale=5.0,
    enable_temporal_reasoning=False,
    num_temporal_reasoning_steps=0,
).frames[0]
Image.fromarray((output[-1] * 255).clip(0, 255).astype("uint8")).save("output.png")

可选地，启用 时间推理 以提升物理一致性：

output = pipe(
    image=image,
    prompt=prompt,
    height=height,
    width=width,
    num_frames=29,
    num_inference_steps=50,
    guidance_scale=5.0,
    enable_temporal_reasoning=True,
    num_temporal_reasoning_steps=50,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
Image.fromarray((output[-1] * 255).clip(0, 255).astype("uint8")).save("output.png")

使用 8 步蒸馏 Lora 进行推理

import torch
import numpy as np
from diffusers import AutoencoderKLWan, ChronoEditTransformer3DModel, ChronoEditPipeline
from diffusers.schedulers import UniPCMultistepScheduler
from diffusers.utils import export_to_video, load_image
from transformers import CLIPVisionModel
from PIL import Image

model_id = "nvidia/ChronoEdit-14B-Diffusers"
image_encoder = CLIPVisionModel.from_pretrained(model_id, subfolder="image_encoder", dtype=torch.float32)
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", dtype=torch.float32)
transformer = ChronoEditTransformer3DModel.from_pretrained(model_id, subfolder="transformer", dtype=torch.bfloat16)
pipe = ChronoEditPipeline.from_pretrained(model_id, image_encoder=image_encoder, transformer=transformer, vae=vae, dtype=torch.bfloat16)
pipe.load_lora_weights("nvidia/ChronoEdit-14B-Diffusers", weight_name="lora/chronoedit_distill_lora.safetensors", adapter_name="distill")
pipe.fuse_lora(adapter_names=["distill"], lora_scale=1.0)
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=2.0)
pipe.to("cuda")  # or "mps", "xpu", "cpu"

image = load_image(
    "https://huggingface.co/spaces/nvidia/ChronoEdit/resolve/main/examples/3.png"
)
max_area = 720 * 1280
aspect_ratio = image.height / image.width
mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
print("width", width, "height", height)
image = image.resize((width, height))
prompt = (
    "The user wants to transform the image by adding a small, cute mouse sitting inside the floral teacup, enjoying a spa bath. The mouse should appear relaxed and cheerful, with a tiny white bath towel draped over its head like a turban. It should be positioned comfortably in the cup’s liquid, with gentle steam rising around it to blend with the cozy atmosphere. "
    "The mouse’s pose should be natural—perhaps sitting upright with paws resting lightly on the rim or submerged in the tea. The teacup’s floral design, gold trim, and warm lighting must remain unchanged to preserve the original aesthetic. The steam should softly swirl around the mouse, enhancing the spa-like, whimsical mood."
)

output = pipe(
    image=image,
    prompt=prompt,
    height=height,
    width=width,
    num_frames=5,
    num_inference_steps=8,
    guidance_scale=1.0,
    enable_temporal_reasoning=False,
    num_temporal_reasoning_steps=0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
Image.fromarray((output[-1] * 255).clip(0, 255).astype("uint8")).save("output.png")

使用多个 LoRA 进行推理

import torch
import numpy as np
from diffusers import AutoencoderKLWan, ChronoEditTransformer3DModel, ChronoEditPipeline
from diffusers.schedulers import UniPCMultistepScheduler
from diffusers.utils import export_to_video, load_image
from transformers import CLIPVisionModel
from PIL import Image

model_id = "nvidia/ChronoEdit-14B-Diffusers"
image_encoder = CLIPVisionModel.from_pretrained(model_id, subfolder="image_encoder", dtype=torch.float32)
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", dtype=torch.float32)
transformer = ChronoEditTransformer3DModel.from_pretrained(model_id, subfolder="transformer", dtype=torch.bfloat16)
pipe = ChronoEditPipeline.from_pretrained(model_id, image_encoder=image_encoder, transformer=transformer, vae=vae, dtype=torch.bfloat16)
pipe.load_lora_weights("nvidia/ChronoEdit-14B-Diffusers-Paint-Brush-Lora", weight_name="paintbrush_lora_diffusers.safetensors", adapter_name="paintbrush")
pipe.load_lora_weights("nvidia/ChronoEdit-14B-Diffusers", weight_name="lora/chronoedit_distill_lora.safetensors", adapter_name="distill")
pipe.fuse_lora(adapter_names=["paintbrush", "distill"], lora_scale=1.0)
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=2.0)
pipe.to("cuda")  # or "mps", "xpu", "cpu"

image = load_image(
    "https://raw.githubusercontent.com/nv-tlabs/ChronoEdit/refs/heads/main/assets/images/input_paintbrush.png"
)
max_area = 720 * 1280
aspect_ratio = image.height / image.width
mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
print("width", width, "height", height)
image = image.resize((width, height))
prompt = (
    "Turn the pencil sketch in the image into an actual object that is consistent with the image’s content. The user wants to change the sketch to a crown and a hat."
)

output = pipe(
    image=image,
    prompt=prompt,
    height=height,
    width=width,
    num_frames=5,
    num_inference_steps=8,
    guidance_scale=1.0,
    enable_temporal_reasoning=False,
    num_temporal_reasoning_steps=0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
Image.fromarray((output[-1] * 255).clip(0, 255).astype("uint8")).save("output_1.png")

ChronoEditPipeline

类 diffusers.ChronoEditPipeline

< source >

( tokenizer: AutoTokenizertext_encoder: UMT5EncoderModelimage_encoder: CLIPVisionModelimage_processor: CLIPImageProcessorPiltransformer: ChronoEditTransformer3DModelvae: AutoencoderKLWanscheduler: FlowMatchEulerDiscreteScheduler )

参数

tokenizer (T5Tokenizer) — 来自 T5 的分词器，具体为 google/umt5-xxl 变体。
text_encoder（T5EncoderModel）—— T5，具体为 google/umt5-xxl 变体。
image_encoder (CLIPVisionModel) — CLIP，具体为 clip-vit-huge-patch14 变体。
transformer (WanTransformer3DModel) — 用于对输入潜变量进行去噪的条件 Transformer。
scheduler (UniPCMultistepScheduler) — 用于与 transformer 配合，对编码后的图像潜变量进行去噪的调度器。
vae (AutoencoderKLWan) — 变分自编码器（VAE）模型，用于将视频编码为潜在表示，以及从潜在表示解码为视频。

使用 Wan 进行图像到视频生成的 Pipeline。

该模型继承自 DiffusionPipeline。有关为所有 pipeline 实现的通用方法（下载、保存、在特定设备上运行等），请参阅父类文档。

call

< source >

( image: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, list[PIL.Image.Image], list[numpy.ndarray], list[torch.Tensor]]prompt: str | list[str] = Nonenegative_prompt: str | list[str] = Noneheight: int = 480width: int = 832num_frames: int = 81num_inference_steps: int = 50guidance_scale: float = 5.0num_videos_per_prompt: int | None = 1generator: typing.Union[torch.Generator, list[torch.Generator], NoneType] = Nonelatents: typing.Optional[torch.Tensor] = Noneprompt_embeds: typing.Optional[torch.Tensor] = Nonenegative_prompt_embeds: typing.Optional[torch.Tensor] = Noneimage_embeds: typing.Optional[torch.Tensor] = Noneoutput_type: str | None = 'np'return_dict: bool = Trueattention_kwargs: dict[str, typing.Any] | None = Nonecallback_on_step_end: typing.Optional[typing.Callable[[int, int, NoneType], diffusers.callbacks.PipelineCallback | diffusers.callbacks.MultiPipelineCallbacks]] = Nonecallback_on_step_end_tensor_inputs: list = ['latents']max_sequence_length: int = 512enable_temporal_reasoning: bool = Falsenum_temporal_reasoning_steps: int = 0 ) → ~ChronoEditPipelineOutput or tuple

参数

image (PipelineImageInput) — 用于条件生成的输入图像。必须为图像、图像列表或 torch.Tensor。
prompt (str 或 list[str]，可选) — 用于引导图像生成的提示词或提示词列表。如果未定义，则必须改为传入 prompt_embeds。
negative_prompt (str 或 list[str]，可选) — 用于不引导图像生成的提示词。如果未定义，则必须传入 negative_prompt_embeds。在未使用引导时会被忽略（即当 guidance_scale 小于 1 时忽略）。
height (int，默认为 480) — 生成视频的高度。
width (int，默认为 832) — 生成视频的宽度。
num_frames (int，默认为 81) — 生成视频的帧数。
num_inference_steps (int，默认为 50) — 去噪步数。更多的去噪步数通常能生成更高质量的图像，但推理速度会变慢。
guidance_scale (float，默认为 5.0) — 如 Classifier-Free Diffusion Guidance 中定义的引导尺度。 guidance_scale 定义为 Imagen Paper 中公式 2 的 w。通过设置 guidance_scale > 1 来启用引导尺度。较高的引导尺度会促使生成与文本 prompt 紧密关联的图像，通常以牺牲图像质量为代价。
num_videos_per_prompt (int，可选，默认为 1) — 每个提示词生成的图像数量。
generator (torch.Generator 或 list[torch.Generator]，可选) — 用于使生成过程确定性的 torch.Generator。
latents (torch.Tensor, 可选) — 预先生成的、从高斯分布中采样的含噪潜变量，用作图像生成的输入。可用于以不同的提示词微调同一生成过程。如果未提供，则使用提供的随机generator进行采样，生成一个潜变量张量。
prompt_embeds (torch.Tensor, 可选) — 预生成的文本嵌入。可用于轻松调整文本输入（提示词加权）。如果未提供，则从 prompt 输入参数生成文本嵌入。
negative_prompt_embeds (torch.Tensor，可选) — 预生成的文本嵌入。可用于轻松调整文本输入（提示词权重）。如果未提供，则从 negative_prompt 输入参数生成文本嵌入。
image_embeds (torch.Tensor, 可选) — 预生成的图像嵌入。可用于轻松调整图像输入（加权）。如果未提供，则从 image 输入参数生成图像嵌入。
output_type (str，可选，默认为 "np") — 生成图像的输出格式。可选 PIL.Image 或 np.array。
return_dict (bool，可选，默认为 True) — 是否返回 ChronoEditPipelineOutput 而不是普通元组。
attention_kwargs (dict, 可选) — 一个 kwargs 字典，如果指定，将传递给 diffusers.models.attention_processor 中定义的 AttentionProcessor，该处理器在 self.processor 下定义。
callback_on_step_end (Callable、PipelineCallback、MultiPipelineCallbacks、可选) — 在推理过程中，每个去噪步骤结束时调用的函数，或 PipelineCallback 或 MultiPipelineCallbacks 的子类，其参数如下：callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)。callback_kwargs 将包含由 callback_on_step_end_tensor_inputs 指定的所有张量列表。
callback_on_step_end_tensor_inputs (List，可选) — callback_on_step_end 函数的张量输入列表。列表中指定的张量将作为 callback_kwargs 参数传入。您只能包含流水线类的 ._callback_tensor_inputs 属性中列出的变量。
max_sequence_length (int，默认为 512) — 文本编码器的最大序列长度。如果提示词长于该长度，将被截断；如果短于该长度，将被填充至该长度。
enable_temporal_reasoning (bool，可选，默认为 False) — 是否启用时序推理。
num_temporal_reasoning_steps (int，可选，默认为 0) — 启用时序推理的步数。

返回值

~ChronoEditPipelineOutput 或 tuple

如果 return_dict 为 True，则返回 ChronoEditPipelineOutput，否则返回一个 tuple，其中第一个元素是包含生成图像的列表，第二个元素是 bool 列表，指示对应的生成图像是否包含“不适合工作场所”（nsfw）内容。

用于生成的 pipeline 的调用函数。

示例：

>>> import torch
>>> import numpy as np
>>> from diffusers import AutoencoderKLWan, ChronoEditTransformer3DModel, ChronoEditPipeline
>>> from diffusers.utils import export_to_video, load_image
>>> from transformers import CLIPVisionModel

>>> # Available models: nvidia/ChronoEdit-14B-Diffusers
>>> model_id = "nvidia/ChronoEdit-14B-Diffusers"
>>> image_encoder = CLIPVisionModel.from_pretrained(
...     model_id, subfolder="image_encoder", torch_dtype=torch.float32
... )
>>> vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
>>> transformer = ChronoEditTransformer3DModel.from_pretrained(
...     model_id, subfolder="transformer", torch_dtype=torch.bfloat16
... )
>>> pipe = ChronoEditPipeline.from_pretrained(
...     model_id, vae=vae, image_encoder=image_encoder, transformer=transformer, torch_dtype=torch.bfloat16
... )
>>> pipe.to("cuda")

>>> image = load_image("https://huggingface.co/spaces/nvidia/ChronoEdit/resolve/main/examples/3.png")
>>> max_area = 720 * 1280
>>> aspect_ratio = image.height / image.width
>>> mod_value = pipe.vae_scale_factor_spatial * pipe.transformer.config.patch_size[1]
>>> height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
>>> width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
>>> image = image.resize((width, height))
>>> prompt = (
...     "The user wants to transform the image by adding a small, cute mouse sitting inside the floral teacup, enjoying a spa bath. The mouse should appear relaxed and cheerful, with a tiny white bath towel draped over its head like a turban. It should be positioned comfortably in the cup’s liquid, with gentle steam rising around it to blend with the cozy atmosphere. "
...     "The mouse’s pose should be natural—perhaps sitting upright with paws resting lightly on the rim or submerged in the tea. The teacup’s floral design, gold trim, and warm lighting must remain unchanged to preserve the original aesthetic. The steam should softly swirl around the mouse, enhancing the spa-like, whimsical mood."
... )

>>> output = pipe(
...     image=image,
...     prompt=prompt,
...     height=height,
...     width=width,
...     num_frames=5,
...     guidance_scale=5.0,
...     enable_temporal_reasoning=False,
...     num_temporal_reasoning_steps=0,
... ).frames[0]
>>> export_to_video(output, "output.mp4", fps=16)

encode_prompt

< source >

( prompt: str | list[str]negative_prompt: str | list[str] | None = Nonedo_classifier_free_guidance: bool = Truenum_videos_per_prompt: int = 1prompt_embeds: typing.Optional[torch.Tensor] = Nonenegative_prompt_embeds: typing.Optional[torch.Tensor] = Nonemax_sequence_length: int = 226device: typing.Optional[torch.device] = Nonedtype: typing.Optional[torch.dtype] = None )

参数

prompt (str 或 list[str]，可选) — 要编码的提示
negative_prompt (str 或 list[str]，可选) — 用于不引导图像生成的提示词。如果未定义，则必须传入 negative_prompt_embeds。在未使用引导时会被忽略（即当 guidance_scale 小于 1 时忽略）。
do_classifier_free_guidance（bool，可选，默认为 True）— 是否使用 classifier free guidance。
num_videos_per_prompt (int，可选，默认为 1) — 每个提示词应生成的视频数量。用于放置结果嵌入的 torch 设备
prompt_embeds (torch.Tensor, 可选) — 预生成的文本嵌入。可用于轻松调整文本输入，例如提示词加权。如果未提供，文本嵌入将从 prompt 输入参数生成。
negative_prompt_embeds (torch.Tensor, 可选) — 预生成的负向文本嵌入。可用于轻松调整文本输入，例如提示词加权。如果未提供，negative_prompt_embeds 将从 negative_prompt 输入参数生成。
device — (torch.device，可选)： torch 设备
dtype — (torch.dtype，可选)： torch 数据类型

将提示词编码为文本编码器的隐藏状态。

ChronoEditPipelineOutput

class diffusers.pipelines.chronoedit.pipeline_output.ChronoEditPipelineOutput

< source >

( frames: Tensor )

参数

frames (torch.Tensor、np.ndarray 或 list[list[PIL.Image.Image]]) — 视频输出列表 - 它可以是一个长度为 batch_size, 的嵌套列表，其中每个子列表包含长度为 num_frames. 的去噪 PIL 图像序列。它也可以是一个形状为 (batch_size, num_frames, channels, height, width) 的 NumPy 数组或 Torch 张量。

ChronoEdit 管道的输出类。

Update on GitHub

Diffusers

ChronoEdit

图像编辑

使用 8 步蒸馏 Lora 进行推理

使用多个 LoRA 进行推理

ChronoEditPipeline

类 diffusers.ChronoEditPipeline

__call__

encode_prompt

ChronoEditPipelineOutput

class diffusers.pipelines.chronoedit.pipeline_output.ChronoEditPipelineOutput

call