File size: 5,339 Bytes
acecd1b c3c5a50 b207b64 c3c5a50 b207b64 c3c5a50 acecd1b c3c5a50 acecd1b c3c5a50 acecd1b c3c5a50 acecd1b c3c5a50 acecd1b c3c5a50 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
import gradio as gr
from pathlib import Path
from scripts.inference import main
from omegaconf import OmegaConf
import argparse
from datetime import datetime
# Download models
from huggingface_hub import snapshot_download
import os
os.makedirs("checkpoints", exist_ok=True)
snapshot_download(
repo_id = "chunyu-li/LatentSync",
local_dir = "./checkpoints"
)
CONFIG_PATH = Path("configs/unet/second_stage.yaml")
CHECKPOINT_PATH = Path("checkpoints/latentsync_unet.pt")
def process_video(
video_path,
audio_path,
guidance_scale,
inference_steps,
seed,
):
# Create the temp directory if it doesn't exist
output_dir = Path("./temp")
output_dir.mkdir(parents=True, exist_ok=True)
# Convert paths to absolute Path objects and normalize them
video_file_path = Path(video_path)
video_path = video_file_path.absolute().as_posix()
audio_path = Path(audio_path).absolute().as_posix()
current_time = datetime.now().strftime("%Y%m%d_%H%M%S")
# Set the output path for the processed video
output_path = str(output_dir / f"{video_file_path.stem}_{current_time}.mp4") # Change the filename as needed
config = OmegaConf.load(CONFIG_PATH)
config["run"].update(
{
"guidance_scale": guidance_scale,
"inference_steps": inference_steps,
}
)
# Parse the arguments
args = create_args(video_path, audio_path, output_path, inference_steps, guidance_scale, seed)
try:
result = main(
config=config,
args=args,
)
print("Processing completed successfully.")
return output_path # Ensure the output path is returned
except Exception as e:
print(f"Error during processing: {str(e)}")
raise gr.Error(f"Error during processing: {str(e)}")
def create_args(
video_path: str, audio_path: str, output_path: str, inference_steps: int, guidance_scale: float, seed: int
) -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--inference_ckpt_path", type=str, required=True)
parser.add_argument("--video_path", type=str, required=True)
parser.add_argument("--audio_path", type=str, required=True)
parser.add_argument("--video_out_path", type=str, required=True)
parser.add_argument("--inference_steps", type=int, default=20)
parser.add_argument("--guidance_scale", type=float, default=1.0)
parser.add_argument("--seed", type=int, default=1247)
return parser.parse_args(
[
"--inference_ckpt_path",
CHECKPOINT_PATH.absolute().as_posix(),
"--video_path",
video_path,
"--audio_path",
audio_path,
"--video_out_path",
output_path,
"--inference_steps",
str(inference_steps),
"--guidance_scale",
str(guidance_scale),
"--seed",
str(seed),
]
)
# Create Gradio interface
with gr.Blocks(title="LatentSync Video Processing") as demo:
gr.Markdown(
"""
# LatentSync: Audio Conditioned Latent Diffusion Models for Lip Sync
Upload a video and audio file to process with LatentSync model.
<div align="center">
<strong>Chunyu Li1,2 Chao Zhang1 Weikai Xu1 Jinghui Xie1,† Weiguo Feng1
Bingyue Peng1 Weiwei Xing2,†</strong>
</div>
<div align="center">
<strong>1ByteDance 2Beijing Jiaotong University</strong>
</div>
<div style="display:flex;justify-content:center;column-gap:4px;">
<a href="https://github.com/bytedance/LatentSync">
<img src='https://img.shields.io/badge/GitHub-Repo-blue'>
</a>
<a href="https://arxiv.org/pdf/2412.09262">
<img src='https://img.shields.io/badge/ArXiv-Paper-red'>
</a>
</div>
"""
)
with gr.Row():
with gr.Column():
video_input = gr.Video(label="Input Video")
audio_input = gr.Audio(label="Input Audio", type="filepath")
with gr.Row():
guidance_scale = gr.Slider(
minimum=1.0,
maximum=3.5,
value=1.5,
step=0.5,
label="Guidance Scale",
)
inference_steps = gr.Slider(minimum=10, maximum=50, value=20, step=1, label="Inference Steps")
with gr.Row():
seed = gr.Number(value=1247, label="Random Seed", precision=0)
process_btn = gr.Button("Process Video")
with gr.Column():
video_output = gr.Video(label="Output Video")
gr.Examples(
examples=[
["assets/demo1_video.mp4", "assets/demo1_audio.wav"],
["assets/demo2_video.mp4", "assets/demo2_audio.wav"],
["assets/demo3_video.mp4", "assets/demo3_audio.wav"],
],
inputs=[video_input, audio_input],
)
process_btn.click(
fn=process_video,
inputs=[
video_input,
audio_input,
guidance_scale,
inference_steps,
seed,
],
outputs=video_output,
)
if __name__ == "__main__":
demo.launch(inbrowser=True, share=True) |