Wan-AI

/

Wan2.2-Animate-14B-Diffusers

like 6

Follow

Wan-AI 6.11k

The diffusers version of the Wan2.2-Animate-14B model can be tested from the PR branch as follows:

import torch

from diffusers import WanAnimatePipeline
from diffusers.utils import export_to_video, load_image, load_video

device = "cuda:0"
dtype = torch.bfloat16
model_id = "Wan-AI/Wan2.2-Animate-14B-Diffusers"
pipe = WanAnimatePipeline.from_pretrained(model_id, revision="/pr/1", torch_dtype=dtype)
pipe.to(device)

seed = 42
prompt = "People in the video are doing actions."

# Animation
image = load_image("/path/to/animate/reference/image/src_ref.png")
pose_video = load_video("/path/to/animate/pose/video/src_pose.mp4")
face_video = load_video("/path/to/animate/face/video/src_face.mp4")

animate_video = pipe(
    image=image,
    pose_video=pose_video,
    face_video=face_video,
    prompt=prompt,
    mode="animate",
    segment_frame_length=77,  # clip_len in original code
    prev_segment_conditioning_frames=1,  # refert_num in original code
    guidance_scale=1.0,
    num_inference_steps=20,
    generator=torch.Generator(device=device).manual_seed(seed),
).frames[0]
export_to_video(animate_video, "diffusers_animate.mp4", fps=30)

# Replacement
image = load_image("/path/to/replace/reference/image/src_ref.png")
pose_video = load_video("/path/to/replace/pose/video/src_pose.mp4")
face_video = load_video("/path/to/replace/face/video/src_face.mp4")
background_video = load_video("/path/to/replace/background/video/src_bg.mp4")
mask_video = load_video("/path/to/replace/mask/video/src_mask.mp4")

replace_video = pipe(
    image=image,
    pose_video=pose_video,
    face_video=face_video,
    background_video=background_video,
    mask_video=mask_video,
    prompt=prompt,
    mode="replace",
    segment_frame_length=77,  # clip_len in original code
    prev_segment_conditioning_frames=1,  # refert_num in original code
    guidance_scale=1.0,
    num_inference_steps=20,
    generator=torch.Generator(device=device).manual_seed(seed),
).frames[0]
export_to_video(replace_video, "diffusers_replace.mp4", fps=30)

Please replace the paths for the image, pose_video, etc. with appropriate local paths. I have mostly tested with the inputs generated by the official Wan Animate preprocessing pipeline.

revision="/pr/1" --> revision="refs/pr/1"

xx

Waving her hand