Upload 4 files

README.md

@@ -1,82 +1,13 @@
 ---
-title: LTX-Video-Playground # Replace with your app's title
-emoji: 🚀 # Choose an emoji to represent your app
-colorFrom: blue # Choose a color to start the gradient (e.g., blue, red, green)
-colorTo: purple # Choose a color to end the gradient
-sdk: gradio # Specify the SDK, e.g., gradio or streamlit
-sdk_version: "4.44.1" # Specify the SDK version if needed
-app_file: app.py # Name of your main app file
-pinned: false # Set to true if you want to pin this Space
+title: LTX Video Fast
+emoji: 🎥
+colorFrom: yellow
+colorTo: pink
+sdk: gradio
+sdk_version: 5.42.0
+app_file: app.py
+pinned: false
+short_description: ultra-fast video model, LTX 0.9.8 13B distilled
 ---
 
-<div align="center">
-
-# Xora️
-
-</div>
-
-This is the official repository for Xora.
-
-## Table of Contents
-
-- [Introduction](#introduction)
-- [Installation](#installation)
-- [Inference](#inference)
-  - [Inference Code](#inference-code)
-- [Acknowledgement](#acknowledgement)
-
-## Introduction
-
-The performance of Diffusion Transformers is heavily influenced by the number of generated latent pixels (or tokens). In video generation, the token count becomes substantial as the number of frames increases. To address this, we designed a carefully optimized VAE that compresses videos into a smaller number of tokens while utilizing a deeper latent space. This approach enables our model to generate high-quality 768x512 videos at 24 FPS, achieving near real-time speeds.
-
-## Installation
-
-# Setup
-
-The codebase currently uses Python 3.10.5, CUDA version 12.2, and supports PyTorch >= 2.1.2.
-
-```bash
-git clone https://github.com/LightricksResearch/xora-core.git
-cd xora-core
-
-# create env
-python -m venv env
-source env/bin/activate
-python -m pip install -e .\[inference-script\]
-```
-
-Then, download the model from [Hugging Face](https://huggingface.co/Lightricks/Xora)
-
-```python
-from huggingface_hub import snapshot_download
-
-model_path = 'PATH'   # The local directory to save downloaded checkpoint
-snapshot_download("Lightricks/Xora", local_dir=model_path, local_dir_use_symlinks=False, repo_type='model')
-```
-
-## Inference
-
-### Inference Code
-
-To use our model, please follow the inference code in `inference.py` at [https://github.com/LightricksResearch/xora-core/blob/main/inference.py]():
-
-For text-to-video generation:
-
-```bash
-python inference.py --ckpt_dir 'PATH' --prompt "PROMPT" --height HEIGHT --width WIDTH
-```
-
-For image-to-video generation:
-
-```python
-python inference.py --ckpt_dir 'PATH' --prompt "PROMPT" --input_image_path IMAGE_PATH --height HEIGHT --width WIDTH
-
-```
-
-## Acknowledgement
-
-We are grateful for the following awesome projects when implementing Xora:
-
-- [DiT](https://github.com/facebookresearch/DiT) and [PixArt-alpha](https://github.com/PixArt-alpha/PixArt-alpha): vision transformers for image generation.
-
-[//]: # "## Citation"
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,797 +1,521 @@
-import spaces
-from functools import lru_cache
 import gradio as gr
-from gradio_toggle import Toggle
 import torch
-from huggingface_hub import hf_hub_download
-from transformers import CLIPProcessor, CLIPModel
-
-from ltx_video.models.autoencoders.causal_video_autoencoder import CausalVideoAutoencoder
-from ltx_video.models.transformers.transformer3d import Transformer3DModel
-from ltx_video.models.transformers.symmetric_patchifier import SymmetricPatchifier
-from ltx_video.schedulers.rf import RectifiedFlowScheduler
-from ltx_video.pipelines.pipeline_ltx_video import  LTXVideoPipeline as XoraVideoPipeline
-from transformers import T5EncoderModel, T5Tokenizer
-from ltx_video.utils.conditioning_method import ConditioningMethod
-from pathlib import Path
-import safetensors.torch
-import json
+import spaces
 import numpy as np
-import cv2
-from PIL import Image
-import tempfile
+import random
 import os
-import gc
-from openai import OpenAI
-import csv
-from datetime import datetime
-
+import yaml
+from pathlib import Path
+import imageio
+import tempfile
+from PIL import Image
+from huggingface_hub import hf_hub_download
+import shutil
+
+from inference import (
+    create_ltx_video_pipeline,
+    create_latent_upsampler,
+    load_image_to_tensor_with_resize_and_crop,
+    seed_everething,
+    get_device,
+    calculate_padding,
+    load_media_file
+)
+from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem, LTXMultiScalePipeline, LTXVideoPipeline
 from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
 
+config_file_path = "configs/ltxv-13b-0.9.8-distilled.yaml"
+with open(config_file_path, "r") as file:
+    PIPELINE_CONFIG_YAML = yaml.safe_load(file)
+
+LTX_REPO = "Lightricks/LTX-Video"
+MAX_IMAGE_SIZE = PIPELINE_CONFIG_YAML.get("max_resolution", 1280)
+MAX_NUM_FRAMES = 257
+
+FPS = 30.0 
+
+# --- Global variables for loaded models ---
+pipeline_instance = None
+latent_upsampler_instance = None
+models_dir = "downloaded_models_gradio_cpu_init"
+Path(models_dir).mkdir(parents=True, exist_ok=True)
+
+print("Downloading models (if not present)...")
+distilled_model_actual_path = hf_hub_download(
+    repo_id=LTX_REPO,
+    filename=PIPELINE_CONFIG_YAML["checkpoint_path"],
+    local_dir=models_dir,
+    local_dir_use_symlinks=False
+)
+PIPELINE_CONFIG_YAML["checkpoint_path"] = distilled_model_actual_path
+print(f"Distilled model path: {distilled_model_actual_path}")
+
+SPATIAL_UPSCALER_FILENAME = PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"]
+spatial_upscaler_actual_path = hf_hub_download(
+    repo_id=LTX_REPO,
+    filename=SPATIAL_UPSCALER_FILENAME,
+    local_dir=models_dir,
+    local_dir_use_symlinks=False
+)
+PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"] = spatial_upscaler_actual_path
+print(f"Spatial upscaler model path: {spatial_upscaler_actual_path}")
+
+print("Creating LTX Video pipeline on CPU...")
+pipeline_instance = create_ltx_video_pipeline(
+    ckpt_path=PIPELINE_CONFIG_YAML["checkpoint_path"],
+    precision=PIPELINE_CONFIG_YAML["precision"],
+    text_encoder_model_name_or_path=PIPELINE_CONFIG_YAML["text_encoder_model_name_or_path"],
+    sampler=PIPELINE_CONFIG_YAML["sampler"],
+    device="cpu",
+    enhance_prompt=False,
+    prompt_enhancer_image_caption_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_image_caption_model_name_or_path"],
+    prompt_enhancer_llm_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_llm_model_name_or_path"],
+)
+print("LTX Video pipeline created on CPU.")
+
+if PIPELINE_CONFIG_YAML.get("spatial_upscaler_model_path"):
+    print("Creating latent upsampler on CPU...")
+    latent_upsampler_instance = create_latent_upsampler(
+        PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"],
+        device="cpu"
+    )
+    print("Latent upsampler created on CPU.")
 
-# Load Hugging Face token if needed
-hf_token = os.getenv("HF_TOKEN")
-openai_api_key = os.getenv("OPENAI_API_KEY")
-client = OpenAI(api_key=openai_api_key)
-system_prompt_t2v_path = "assets/system_prompt_t2v.txt"
-system_prompt_i2v_path = "assets/system_prompt_i2v.txt"
-with open(system_prompt_t2v_path, "r") as f:
-    system_prompt_t2v = f.read()
-
-with open(system_prompt_i2v_path, "r") as f:
-    system_prompt_i2v = f.read()
-
-# Set model download directory within Hugging Face Spaces
-model_path = Path("/home/elevin/xora-core/assets/")
-cpkt_path = Path("/home/elevin/xora-core/assets/ltx-video-2b-v0.9.1.safetensors")
-if not os.path.exists(cpkt_path):
-    hf_hub_download(repo_id="Lightricks/LTX-Video", filename="ltx-video-2b-v0.9.1.safetensors", local_dir=model_path, local_dir_use_symlinks=False, repo_type='model')
-# Global variables to load components
-vae_dir = Path(model_path) / "vae"
-unet_dir = Path(model_path) / "unet"
-scheduler_dir = Path(model_path) / "scheduler"
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-DATA_DIR = "/data"
-os.makedirs(DATA_DIR, exist_ok=True)
-LOG_FILE_PATH = os.path.join("/data", "user_requests.csv")
-
-clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32", cache_dir=model_path)
-clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32", cache_dir=model_path)
-
-
-if not os.path.exists(LOG_FILE_PATH):
-    with open(LOG_FILE_PATH, "w", newline="") as f:
-        writer = csv.writer(f)
-        writer.writerow(
-            [
-                "timestamp",
-                "request_type",
-                "prompt",
-                "negative_prompt",
-                "height",
-                "width",
-                "num_frames",
-                "frame_rate",
-                "seed",
-                "num_inference_steps",
-                "guidance_scale",
-                "is_enhanced",
-                "clip_embedding",
-                "original_resolution",
-            ]
-        )
-
-
-@lru_cache(maxsize=128)
-def log_request(
-    request_type,
-    prompt,
-    negative_prompt,
-    height,
-    width,
-    num_frames,
-    frame_rate,
-    seed,
-    num_inference_steps,
-    guidance_scale,
-    is_enhanced,
-    clip_embedding=None,
-    original_resolution=None,
-):
-    """Log the user's request to a CSV file."""
-    timestamp = datetime.now().isoformat()
-    with open(LOG_FILE_PATH, "a", newline="") as f:
-        try:
-            writer = csv.writer(f)
-            writer.writerow(
-                [
-                    timestamp,
-                    request_type,
-                    prompt,
-                    negative_prompt,
-                    height,
-                    width,
-                    num_frames,
-                    frame_rate,
-                    seed,
-                    num_inference_steps,
-                    guidance_scale,
-                    is_enhanced,
-                    clip_embedding,
-                    original_resolution,
-                ]
-            )
-        except Exception as e:
-            print(f"Error logging request: {e}")
+target_inference_device = "cuda"
+print(f"Target inference device: {target_inference_device}")
+pipeline_instance.to(target_inference_device)
+if latent_upsampler_instance: 
+    latent_upsampler_instance.to(target_inference_device)
 
 
-def compute_clip_embedding(text=None, image=None):
+# --- Helper function for dimension calculation ---
+MIN_DIM_SLIDER = 256  # As defined in the sliders minimum attribute
+TARGET_FIXED_SIDE = 768 # Desired fixed side length as per requirement
+
+def calculate_new_dimensions(orig_w, orig_h):
     """
-    Compute CLIP embedding for a given text or image.
-    Args:
-        text (str): Input text prompt.
-        image (PIL.Image): Input image.
-    Returns:
-        list: CLIP embedding as a list of floats.
+    Calculates new dimensions for height and width sliders based on original media dimensions.
+    Ensures one side is TARGET_FIXED_SIDE, the other is scaled proportionally,
+    both are multiples of 32, and within [MIN_DIM_SLIDER, MAX_IMAGE_SIZE].
     """
-    inputs = clip_processor(text=text, images=image, return_tensors="pt", padding=True)
-    outputs = clip_model.get_text_features(**inputs) if text else clip_model.get_image_features(**inputs)
-    embedding = outputs.detach().cpu().numpy().flatten().tolist()
-    return embedding
-
-
-def load_vae(vae_dir):
-    return CausalVideoAutoencoder.from_pretrained(cpkt_path).to(device=device, dtype=torch.bfloat16)
-
-
-def load_unet(unet_dir):
-    return Transformer3DModel.from_pretrained(cpkt_path).to(device=device, dtype=torch.bfloat16)
-
-
-def load_scheduler(scheduler_dir):
-    return RectifiedFlowScheduler.from_pretrained(cpkt_path)
+    if orig_w == 0 or orig_h == 0:
+        # Default to TARGET_FIXED_SIDE square if original dimensions are invalid
+        return int(TARGET_FIXED_SIDE), int(TARGET_FIXED_SIDE)
+
+    if orig_w >= orig_h:  # Landscape or square
+        new_h = TARGET_FIXED_SIDE
+        aspect_ratio = orig_w / orig_h
+        new_w_ideal = new_h * aspect_ratio
+        
+        # Round to nearest multiple of 32
+        new_w = round(new_w_ideal / 32) * 32
+        
+        # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
+        new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
+        # Ensure new_h is also clamped (TARGET_FIXED_SIDE should be within these bounds if configured correctly)
+        new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE)) 
+    else:  # Portrait
+        new_w = TARGET_FIXED_SIDE
+        aspect_ratio = orig_h / orig_w # Use H/W ratio for portrait scaling
+        new_h_ideal = new_w * aspect_ratio
+        
+        # Round to nearest multiple of 32
+        new_h = round(new_h_ideal / 32) * 32
+        
+        # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
+        new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE))
+        # Ensure new_w is also clamped
+        new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
+
+    return int(new_h), int(new_w)
+
+def get_duration(prompt, negative_prompt, input_image_filepath, input_video_filepath,
+             height_ui, width_ui, mode,
+             duration_ui, # Removed ui_steps
+             ui_frames_to_use,
+             seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
+             progress):
+    if duration_ui > 7:
+        return 75
+    else:
+        return 60
+
+@spaces.GPU(duration=get_duration)
+def generate(prompt, negative_prompt, input_image_filepath=None, input_video_filepath=None,
+             height_ui=512, width_ui=704, mode="text-to-video",
+             duration_ui=2.0, 
+             ui_frames_to_use=9,
+             seed_ui=42, randomize_seed=True, ui_guidance_scale=3.0, improve_texture_flag=True,
+             progress=gr.Progress(track_tqdm=True)):
+    """
+    Generate high-quality videos using LTX Video model with support for text-to-video, image-to-video, and video-to-video modes.
 
+    Args:
+        prompt (str): Text description of the desired video content. Required for all modes.
+        negative_prompt (str): Text describing what to avoid in the generated video. Optional, can be empty string.
+        input_image_filepath (str or None): Path to input image file. Required for image-to-video mode, None for other modes.
+        input_video_filepath (str or None): Path to input video file. Required for video-to-video mode, None for other modes.
+        height_ui (int): Height of the output video in pixels, must be divisible by 32. Default: 512.
+        width_ui (int): Width of the output video in pixels, must be divisible by 32. Default: 704.
+        mode (str): Generation mode. Required. One of "text-to-video", "image-to-video", or "video-to-video". Default: "text-to-video".
+        duration_ui (float): Duration of the output video in seconds. Range: 0.3 to 8.5. Default: 2.0.
+        ui_frames_to_use (int): Number of frames to use from input video. Only used in video-to-video mode. Must be N*8+1. Default: 9.
+        seed_ui (int): Random seed for reproducible generation. Range: 0 to 2^32-1. Default: 42.
+        randomize_seed (bool): Whether to use a random seed instead of seed_ui. Default: True.
+        ui_guidance_scale (float): CFG scale controlling prompt influence. Range: 1.0 to 10.0. Higher values = stronger prompt influence. Default: 3.0.
+        improve_texture_flag (bool): Whether to use multi-scale generation for better texture quality. Slower but higher quality. Default: True.
+        progress (gr.Progress): Progress tracker for the generation process. Optional, used for UI updates.
 
-# Helper function for image processing
-def center_crop_and_resize(frame, target_height, target_width):
-    h, w, _ = frame.shape
-    aspect_ratio_target = target_width / target_height
-    aspect_ratio_frame = w / h
-    if aspect_ratio_frame > aspect_ratio_target:
-        new_width = int(h * aspect_ratio_target)
-        x_start = (w - new_width) // 2
-        frame_cropped = frame[:, x_start : x_start + new_width]
-    else:
-        new_height = int(w / aspect_ratio_target)
-        y_start = (h - new_height) // 2
-        frame_cropped = frame[y_start : y_start + new_height, :]
-    frame_resized = cv2.resize(frame_cropped, (target_width, target_height))
-    return frame_resized
-
-
-def load_image_to_tensor_with_resize(image_path, target_height=512, target_width=768):
-    image = Image.open(image_path).convert("RGB")
-    image_np = np.array(image)
-    frame_resized = center_crop_and_resize(image_np, target_height, target_width)
-    frame_tensor = torch.tensor(frame_resized).permute(2, 0, 1).float()
-    frame_tensor = (frame_tensor / 127.5) - 1.0
-    return frame_tensor.unsqueeze(0).unsqueeze(2)
-
-
-def enhance_prompt_if_enabled(prompt, enhance_toggle, type="t2v"):
-    if not enhance_toggle:
-        print("Enhance toggle is off, Prompt: ", prompt)
-        return prompt
-
-    system_prompt = system_prompt_t2v if type == "t2v" else system_prompt_i2v
-    messages = [
-        {"role": "system", "content": system_prompt},
-        {"role": "user", "content": prompt},
-    ]
+    Returns:
+        tuple: A tuple containing (output_video_path, used_seed) where output_video_path is the path to the generated video file and used_seed is the actual seed used for generation.
+    """
 
-    try:
-        response = client.chat.completions.create(
-            model="gpt-4o-mini",
-            messages=messages,
-            max_tokens=200,
-        )
-        print("Enhanced Prompt: ", response.choices[0].message.content.strip())
-        return response.choices[0].message.content.strip()
-    except Exception as e:
-        print(f"Error: {e}")
-        return prompt
-
-
-# Preset options for resolution and frame configuration
-preset_options = [
-    {"label": "1216x704, 41 frames", "width": 1216, "height": 704, "num_frames": 41},
-    {"label": "1088x704, 49 frames", "width": 1088, "height": 704, "num_frames": 49},
-    {"label": "1056x640, 57 frames", "width": 1056, "height": 640, "num_frames": 57},
-    {"label": "992x608, 65 frames", "width": 992, "height": 608, "num_frames": 65},
-    {"label": "896x608, 73 frames", "width": 896, "height": 608, "num_frames": 73},
-    {"label": "896x544, 81 frames", "width": 896, "height": 544, "num_frames": 81},
-    {"label": "832x544, 89 frames", "width": 832, "height": 544, "num_frames": 89},
-    {"label": "800x512, 97 frames", "width": 800, "height": 512, "num_frames": 97},
-    {"label": "768x512, 97 frames", "width": 768, "height": 512, "num_frames": 97},
-    {"label": "800x480, 105 frames", "width": 800, "height": 480, "num_frames": 105},
-    {"label": "736x480, 113 frames", "width": 736, "height": 480, "num_frames": 113},
-    {"label": "704x480, 121 frames", "width": 704, "height": 480, "num_frames": 121},
-    {"label": "704x448, 129 frames", "width": 704, "height": 448, "num_frames": 129},
-    {"label": "672x448, 137 frames", "width": 672, "height": 448, "num_frames": 137},
-    {"label": "640x416, 153 frames", "width": 640, "height": 416, "num_frames": 153},
-    {"label": "672x384, 161 frames", "width": 672, "height": 384, "num_frames": 161},
-    {"label": "640x384, 169 frames", "width": 640, "height": 384, "num_frames": 169},
-    {"label": "608x384, 177 frames", "width": 608, "height": 384, "num_frames": 177},
-    {"label": "576x384, 185 frames", "width": 576, "height": 384, "num_frames": 185},
-    {"label": "608x352, 193 frames", "width": 608, "height": 352, "num_frames": 193},
-    {"label": "576x352, 201 frames", "width": 576, "height": 352, "num_frames": 201},
-    {"label": "544x352, 209 frames", "width": 544, "height": 352, "num_frames": 209},
-    {"label": "512x352, 225 frames", "width": 512, "height": 352, "num_frames": 225},
-    {"label": "512x352, 233 frames", "width": 512, "height": 352, "num_frames": 233},
-    {"label": "544x320, 241 frames", "width": 544, "height": 320, "num_frames": 241},
-    {"label": "512x320, 249 frames", "width": 512, "height": 320, "num_frames": 249},
-    {"label": "512x320, 257 frames", "width": 512, "height": 320, "num_frames": 257},
-]
-
-
-# Function to toggle visibility of sliders based on preset selection
-def preset_changed(preset):
-    if preset != "Custom":
-        selected = next(item for item in preset_options if item["label"] == preset)
-        return (
-            selected["height"],
-            selected["width"],
-            selected["num_frames"],
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-        )
+    # Validate mode-specific required parameters
+    if mode == "image-to-video":
+        if not input_image_filepath:
+            raise gr.Error("input_image_filepath is required for image-to-video mode")
+    elif mode == "video-to-video":
+        if not input_video_filepath:
+            raise gr.Error("input_video_filepath is required for video-to-video mode")
+    elif mode == "text-to-video":
+        # No additional file inputs required for text-to-video
+        pass
     else:
-        return (
-            None,
-            None,
-            None,
-            gr.update(visible=True),
-            gr.update(visible=True),
-            gr.update(visible=True),
-        )
-
-
-# Load models
-vae = load_vae(vae_dir)
-unet = load_unet(unet_dir)
-scheduler = load_scheduler(scheduler_dir)
-patchifier = SymmetricPatchifier(patch_size=1)
-text_encoder = T5EncoderModel.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="text_encoder").to(device)
-tokenizer = T5Tokenizer.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="tokenizer")
-
-pipeline = XoraVideoPipeline(
-    transformer=unet,
-    patchifier=patchifier,
-    text_encoder=text_encoder,
-    tokenizer=tokenizer,
-    scheduler=scheduler,
-    vae=vae,
-).to(device)
-
-@spaces.GPU(duration=120)
-def generate_video_from_text(
-    prompt="",
-    enhance_prompt_toggle=False,
-    txt2vid_analytics_toggle=True,
-    negative_prompt="",
-    frame_rate=25,
-    seed=646373,
-    num_inference_steps=30,
-    guidance_scale=3,
-    height=512,
-    width=768,
-    num_frames=121,
-    progress=gr.Progress(),
-    stg_scale=1.0,
-    stg_rescale=0.7,
-    stg_mode="stg_a",
-    stg_skip_layers="19",
-):
-    if len(prompt.strip()) < 50:
-        raise gr.Error(
-            "Prompt must be at least 50 characters long. Please provide more details for the best results.",
-            duration=5,
-        )
-
-    if txt2vid_analytics_toggle:
-        log_request(
-            "txt2vid",
-            prompt,
-            negative_prompt,
-            height,
-            width,
-            num_frames,
-            frame_rate,
-            seed,
-            num_inference_steps,
-            guidance_scale,
-            enhance_prompt_toggle,
-        )
-
-    prompt = enhance_prompt_if_enabled(prompt, enhance_prompt_toggle, type="t2v")
-
-    sample = {
+        raise gr.Error(f"Invalid mode: {mode}. Must be one of: text-to-video, image-to-video, video-to-video")
+
+    if randomize_seed:
+        seed_ui = random.randint(0, 2**32 - 1)
+    seed_everething(int(seed_ui))
+    
+    target_frames_ideal = duration_ui * FPS
+    target_frames_rounded = round(target_frames_ideal)
+    if target_frames_rounded < 1: 
+        target_frames_rounded = 1
+    
+    n_val = round((float(target_frames_rounded) - 1.0) / 8.0)
+    actual_num_frames = int(n_val * 8 + 1)
+
+    actual_num_frames = max(9, actual_num_frames)
+    actual_num_frames = min(MAX_NUM_FRAMES, actual_num_frames)
+    
+    actual_height = int(height_ui)
+    actual_width = int(width_ui)
+
+    height_padded = ((actual_height - 1) // 32 + 1) * 32
+    width_padded = ((actual_width - 1) // 32 + 1) * 32
+    num_frames_padded = ((actual_num_frames - 2) // 8 + 1) * 8 + 1 
+    if num_frames_padded != actual_num_frames:
+        print(f"Warning: actual_num_frames ({actual_num_frames}) and num_frames_padded ({num_frames_padded}) differ. Using num_frames_padded for pipeline.")
+    
+    padding_values = calculate_padding(actual_height, actual_width, height_padded, width_padded)
+
+    call_kwargs = {
         "prompt": prompt,
-        "prompt_attention_mask": None,
         "negative_prompt": negative_prompt,
-        "negative_prompt_attention_mask": None,
+        "height": height_padded,
+        "width": width_padded,
+        "num_frames": num_frames_padded, 
+        "frame_rate": int(FPS), 
+        "generator": torch.Generator(device=target_inference_device).manual_seed(int(seed_ui)),
+        "output_type": "pt", 
+        "conditioning_items": None,
         "media_items": None,
+        "decode_timestep": PIPELINE_CONFIG_YAML["decode_timestep"],
+        "decode_noise_scale": PIPELINE_CONFIG_YAML["decode_noise_scale"],
+        "stochastic_sampling": PIPELINE_CONFIG_YAML["stochastic_sampling"],
+        "image_cond_noise_scale": 0.15,
+        "is_video": True,
+        "vae_per_channel_normalize": True,
+        "mixed_precision": (PIPELINE_CONFIG_YAML["precision"] == "mixed_precision"),
+        "offload_to_cpu": False,
+        "enhance_prompt": False,
     }
 
-    generator = torch.Generator(device=device).manual_seed(seed)
+    stg_mode_str = PIPELINE_CONFIG_YAML.get("stg_mode", "attention_values")
+    if stg_mode_str.lower() in ["stg_av", "attention_values"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionValues
+    elif stg_mode_str.lower() in ["stg_as", "attention_skip"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionSkip
+    elif stg_mode_str.lower() in ["stg_r", "residual"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.Residual
+    elif stg_mode_str.lower() in ["stg_t", "transformer_block"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.TransformerBlock
+    else:
+        raise ValueError(f"Invalid stg_mode: {stg_mode_str}")
 
-    def gradio_progress_callback(self, step, timestep, kwargs):
-        progress((step + 1) / num_inference_steps)
+    if mode == "image-to-video" and input_image_filepath:
+        try:
+            media_tensor = load_image_to_tensor_with_resize_and_crop(
+                input_image_filepath, actual_height, actual_width
+            )
+            media_tensor = torch.nn.functional.pad(media_tensor, padding_values)
+            call_kwargs["conditioning_items"] = [ConditioningItem(media_tensor.to(target_inference_device), 0, 1.0)]
+        except Exception as e:
+            print(f"Error loading image {input_image_filepath}: {e}")
+            raise gr.Error(f"Could not load image: {e}")
+    elif mode == "video-to-video" and input_video_filepath:
+        try:
+            call_kwargs["media_items"] = load_media_file(
+                media_path=input_video_filepath,
+                height=actual_height, 
+                width=actual_width,
+                max_frames=int(ui_frames_to_use), 
+                padding=padding_values
+            ).to(target_inference_device)
+        except Exception as e:
+            print(f"Error loading video {input_video_filepath}: {e}")
+            raise gr.Error(f"Could not load video: {e}")
+
+    print(f"Moving models to {target_inference_device} for inference (if not already there)...")
+    
+    active_latent_upsampler = None
+    if improve_texture_flag and latent_upsampler_instance:
+        active_latent_upsampler = latent_upsampler_instance
+
+    result_images_tensor = None
+    if improve_texture_flag:
+        if not active_latent_upsampler:
+            raise gr.Error("Spatial upscaler model not loaded or improve_texture not selected, cannot use multi-scale.")
+        
+        multi_scale_pipeline_obj = LTXMultiScalePipeline(pipeline_instance, active_latent_upsampler)
+        
+        first_pass_args = PIPELINE_CONFIG_YAML.get("first_pass", {}).copy()
+        first_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
+        # num_inference_steps will be derived from len(timesteps) in the pipeline
+        first_pass_args.pop("num_inference_steps", None)
+
+
+        second_pass_args = PIPELINE_CONFIG_YAML.get("second_pass", {}).copy()
+        second_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
+        # num_inference_steps will be derived from len(timesteps) in the pipeline
+        second_pass_args.pop("num_inference_steps", None)
+        
+        multi_scale_call_kwargs = call_kwargs.copy()
+        multi_scale_call_kwargs.update({
+            "downscale_factor": PIPELINE_CONFIG_YAML["downscale_factor"],
+            "first_pass": first_pass_args,
+            "second_pass": second_pass_args,
+        })
+        
+        print(f"Calling multi-scale pipeline (eff. HxW: {actual_height}x{actual_width}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
+        result_images_tensor = multi_scale_pipeline_obj(**multi_scale_call_kwargs).images
+    else:
+        single_pass_call_kwargs = call_kwargs.copy()
+        first_pass_config_from_yaml = PIPELINE_CONFIG_YAML.get("first_pass", {})
+
+        single_pass_call_kwargs["timesteps"] = first_pass_config_from_yaml.get("timesteps")
+        single_pass_call_kwargs["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
+        single_pass_call_kwargs["stg_scale"] = first_pass_config_from_yaml.get("stg_scale")
+        single_pass_call_kwargs["rescaling_scale"] = first_pass_config_from_yaml.get("rescaling_scale")
+        single_pass_call_kwargs["skip_block_list"] = first_pass_config_from_yaml.get("skip_block_list")
+        
+        # Remove keys that might conflict or are not used in single pass / handled by above
+        single_pass_call_kwargs.pop("num_inference_steps", None) 
+        single_pass_call_kwargs.pop("first_pass", None) 
+        single_pass_call_kwargs.pop("second_pass", None)
+        single_pass_call_kwargs.pop("downscale_factor", None)
+        
+        print(f"Calling base pipeline (padded HxW: {height_padded}x{width_padded}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
+        result_images_tensor = pipeline_instance(**single_pass_call_kwargs).images
+
+    if result_images_tensor is None:
+        raise gr.Error("Generation failed.")
+
+    pad_left, pad_right, pad_top, pad_bottom = padding_values
+    slice_h_end = -pad_bottom if pad_bottom > 0 else None
+    slice_w_end = -pad_right if pad_right > 0 else None
+    
+    result_images_tensor = result_images_tensor[
+        :, :, :actual_num_frames, pad_top:slice_h_end, pad_left:slice_w_end
+    ]
 
-    try:
-        with torch.no_grad():
-            images = pipeline(
-                num_inference_steps=num_inference_steps,
-                num_images_per_prompt=1,
-                guidance_scale=guidance_scale,
-                generator=generator,
-                output_type="pt",
-                height=height,
-                width=width,
-                num_frames=num_frames,
-                frame_rate=frame_rate,
-                **sample,
-                is_video=True,
-                vae_per_channel_normalize=True,
-                conditioning_method=ConditioningMethod.UNCONDITIONAL,
-                mixed_precision=True,
-                callback_on_step_end=gradio_progress_callback,
-                stg_scale=stg_scale,
-    do_rescaling=stg_rescale != 1,
-    rescaling_scale=stg_rescale,
-    skip_layer_strategy=SkipLayerStrategy.Attention if stg_mode == "stg_a" else SkipLayerStrategy.Residual,
-    skip_block_list=[int(x.strip()) for x in stg_skip_layers.split(",")]
-            ).images
-    except Exception as e:
-        raise gr.Error(
-            f"An error occurred while generating the video. Please try again. Error: {e}",
-            duration=5,
-        )
-    finally:
-        torch.cuda.empty_cache()
-        gc.collect()
-
-    output_path = tempfile.mktemp(suffix=".mp4")
-    print(images.shape)
-    video_np = images.squeeze(0).permute(1, 2, 3, 0).cpu().float().numpy()
+    video_np = result_images_tensor[0].permute(1, 2, 3, 0).cpu().float().numpy()
+    
+    video_np = np.clip(video_np, 0, 1) 
     video_np = (video_np * 255).astype(np.uint8)
-    height, width = video_np.shape[1:3]
-    out = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*"mp4v"), frame_rate, (width, height))
-    for frame in video_np[..., ::-1]:
-        out.write(frame)
-    out.release()
-    # Explicitly delete tensors and clear cache
-    del images
-    del video_np
-    torch.cuda.empty_cache()
-    return output_path
-
-@spaces.GPU(duration=120)
-def generate_video_from_image(
-    image_path,
-    prompt="",
-    enhance_prompt_toggle=False,
-    img2vid_analytics_toggle=True,
-    negative_prompt="",
-    frame_rate=25,
-    seed=646373,
-    num_inference_steps=30,
-    guidance_scale=3,
-    height=512,
-    width=768,
-    num_frames=121,
-    progress=gr.Progress(),
-    stg_scale=1.0,
-    stg_rescale=0.7,
-    stg_mode="stg_a",
-    stg_skip_layers="19",
-):
-
-    print("Height: ", height)
-    print("Width: ", width)
-    print("Num Frames: ", num_frames)
-
-    if len(prompt.strip()) < 50:
-        raise gr.Error(
-            "Prompt must be at least 50 characters long. Please provide more details for the best results.",
-            duration=5,
-        )
-
-    if not image_path:
-        raise gr.Error("Please provide an input image.", duration=5)
-
-    if img2vid_analytics_toggle:
-        with Image.open(image_path) as img:
-            original_resolution = f"{img.width}x{img.height}"  # Format as "widthxheight"
-            clip_embedding = compute_clip_embedding(image=img)
-
-        log_request(
-            "img2vid",
-            prompt,
-            negative_prompt,
-            height,
-            width,
-            num_frames,
-            frame_rate,
-            seed,
-            num_inference_steps,
-            guidance_scale,
-            enhance_prompt_toggle,
-            json.dumps(clip_embedding),
-            original_resolution,
-        )
-
-    media_items = load_image_to_tensor_with_resize(image_path, height, width).to(device).detach()
-
-    prompt = enhance_prompt_if_enabled(prompt, enhance_prompt_toggle, type="i2v")
-
-    sample = {
-        "prompt": prompt,
-        "prompt_attention_mask": None,
-        "negative_prompt": negative_prompt,
-        "negative_prompt_attention_mask": None,
-        "media_items": media_items,
-    }
-
-    generator = torch.Generator(device=device).manual_seed(seed)
-
-    def gradio_progress_callback(self, step, timestep, kwargs):
-        progress((step + 1) / num_inference_steps)
 
+    temp_dir = tempfile.mkdtemp()
+    timestamp = random.randint(10000,99999)
+    output_video_path = os.path.join(temp_dir, f"output_{timestamp}.mp4")
+    
     try:
-        with torch.no_grad():
-            images = pipeline(
-                num_inference_steps=num_inference_steps,
-                num_images_per_prompt=1,
-                guidance_scale=guidance_scale,
-                generator=generator,
-                output_type="pt",
-                height=height,
-                width=width,
-                num_frames=num_frames,
-                frame_rate=frame_rate,
-                **sample,
-                is_video=True,
-                vae_per_channel_normalize=True,
-                conditioning_method=ConditioningMethod.FIRST_FRAME,
-                mixed_precision=True,
-                callback_on_step_end=gradio_progress_callback,
-                stg_scale=stg_scale,
-                do_rescaling=stg_rescale != 1,
-                rescaling_scale=stg_rescale,
-                skip_layer_strategy=SkipLayerStrategy.Attention if stg_mode == "stg_a" else SkipLayerStrategy.Residual,
-                skip_block_list=[int(x.strip()) for x in stg_skip_layers.split(",")]
-            ).images
-
-        output_path = tempfile.mktemp(suffix=".mp4")
-        video_np = images.squeeze(0).permute(1, 2, 3, 0).cpu().float().numpy()
-        video_np = (video_np * 255).astype(np.uint8)
-        height, width = video_np.shape[1:3]
-        out = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*"mp4v"), frame_rate, (width, height))
-        for frame in video_np[..., ::-1]:
-            out.write(frame)
-        out.release()
+        with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], macro_block_size=1) as video_writer:
+            for frame_idx in range(video_np.shape[0]):
+                progress(frame_idx / video_np.shape[0], desc="Saving video")
+                video_writer.append_data(video_np[frame_idx])
     except Exception as e:
-        raise gr.Error(
-            f"An error occurred while generating the video. Please try again. Error: {e}",
-            duration=5,
-        )
-
-    finally:
-        torch.cuda.empty_cache()
-        gc.collect()
-
-    return output_path
-
-
-def create_advanced_options():
-    with gr.Accordion("Step 4: Advanced Options (Optional)", open=False):
-        seed = gr.Slider(label="4.1 Seed", minimum=0, maximum=1000000, step=1, value=646373)
-        inference_steps = gr.Slider(label="4.2 Inference Steps", minimum=1, maximum=50, step=1, value=30)
-        guidance_scale = gr.Slider(label="4.3 Guidance Scale", minimum=1.0, maximum=5.0, step=0.1, value=3.0)
-
-        height_slider = gr.Slider(
-            label="4.4 Height",
-            minimum=256,
-            maximum=1024,
-            step=64,
-            value=512,
-            visible=False,
-        )
-        width_slider = gr.Slider(
-            label="4.5 Width",
-            minimum=256,
-            maximum=1024,
-            step=64,
-            value=768,
-            visible=False,
-        )
-        num_frames_slider = gr.Slider(
-            label="4.5 Number of Frames",
-            minimum=1,
-            maximum=200,
-            step=1,
-            value=121,
-            visible=False,
-        )
-
-        return [
-            seed,
-            inference_steps,
-            guidance_scale,
-            height_slider,
-            width_slider,
-            num_frames_slider,
-        ]
-
-
-# Define the Gradio interface with tabs
-with gr.Blocks(theme=gr.themes.Soft()) as iface:
-    with gr.Row(elem_id="title-row"):
-        gr.Markdown(
-            """
-        <div style="text-align: center; margin-bottom: 1em">
-            <h1 style="font-size: 2.5em; font-weight: 600; margin: 0.5em 0;">Video Generation with LTX Video</h1>
-        </div>
-        """
-        )
-    with gr.Row(elem_id="title-row"):
-        gr.HTML(  # add technical report link
-            """
-        <div style="display:flex;column-gap:4px;">
-            <a href="https://github.com/Lightricks/LTX-Video">
-                <img src='https://img.shields.io/badge/GitHub-Repo-blue'>
-            </a>
-            <a href="https://github.com/Lightricks/ComfyUI-LTXVideo">
-                <img src='https://img.shields.io/badge/GitHub-ComfyUI-blue'>
-            </a>
-            <a href="http://www.lightricks.com/ltxv">
-                <img src="https://img.shields.io/badge/Project-Page-green" alt="Follow me on HF">
-            </a>
-            <a href="https://huggingface.co/spaces/Lightricks/LTX-Video-Playground?duplicate=true">
-                <img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-sm.svg" alt="Duplicate this Space">
-            </a>
-            <a href="https://huggingface.co/Lightricks">
-                <img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/follow-me-on-HF-sm-dark.svg" alt="Follow me on HF">
-            </a>
-        </div>
-        """
-        )
-    with gr.Accordion(" 📖 Tips for Best Results", open=False, elem_id="instructions-accordion"):
-        gr.Markdown(
-            """
-        📝 Prompt Engineering
-
-        When writing prompts, focus on detailed, chronological descriptions of actions and scenes. Include specific movements, appearances, camera angles, and environmental details - all in a single flowing paragraph. Start directly with the action, and keep descriptions literal and precise. Think like a cinematographer describing a shot list. Keep within 200 words.
-        For best results, build your prompts using this structure:
-
-        - Start with main action in a single sentence
-        - Add specific details about movements and gestures
-        - Describe character/object appearances precisely
-        - Include background and environment details
-        - Specify camera angles and movements
-        - Describe lighting and colors
-        - Note any changes or sudden events
-
-        See examples for more inspiration.
-
-        🎮 Parameter Guide
-
-        - Resolution Preset: Higher resolutions for detailed scenes, lower for faster generation and simpler scenes
-        - Seed: Save seed values to recreate specific styles or compositions you like
-        - Guidance Scale: 3-3.5 are the recommended values
-        - Inference Steps: More steps (40+) for quality, fewer steps (20-30) for speed
-        """
-        )
-
-    with gr.Tabs():
-        # Text to Video Tab
-        with gr.TabItem("Text to Video"):
-            with gr.Row():
-                with gr.Column():
-                    txt2vid_prompt = gr.Textbox(
-                        label="Step 1: Enter Your Prompt",
-                        placeholder="Describe the video you want to generate (minimum 50 characters)...",
-                        value="A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage.",
-                        lines=5,
-                    )
-                    txt2vid_analytics_toggle = Toggle(
-                        label="I agree to share my usage data anonymously to help improve the model features.",
-                        value=True,
-                        interactive=True,
-                    )
-
-                    txt2vid_enhance_toggle = Toggle(
-                        label="Enhance Prompt",
-                        value=False,
-                        interactive=True,
-                    )
-
-                    txt2vid_negative_prompt = gr.Textbox(
-                        label="Step 2: Enter Negative Prompt",
-                        placeholder="Describe what you don't want in the video...",
-                        value="low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                        lines=2,
-                    )
-
-                    txt2vid_preset = gr.Dropdown(
-                        choices=[p["label"] for p in preset_options],
-                        value="768x512, 97 frames",
-                        label="Step 3.1: Choose Resolution Preset",
-                    )
-
-                    txt2vid_frame_rate = gr.Slider(
-                        label="Step 3.2: Frame Rate",
-                        minimum=21,
-                        maximum=30,
-                        step=1,
-                        value=25,
-                    )
-
-                    txt2vid_advanced = create_advanced_options()
-                    txt2vid_generate = gr.Button(
-                        "Step 5: Generate Video",
-                        variant="primary",
-                        size="lg",
-                    )
-
-                with gr.Column():
-                    txt2vid_output = gr.Video(label="Generated Output")
-
-            with gr.Row():
-                gr.Examples(
-                    examples=[
-                        [
-                            "A young woman in a traditional Mongolian dress is peeking through a sheer white curtain, her face showing a mix of curiosity and apprehension. The woman has long black hair styled in two braids, adorned with white beads, and her eyes are wide with a hint of surprise. Her dress is a vibrant blue with intricate gold embroidery, and she wears a matching headband with a similar design. The background is a simple white curtain, which creates a sense of mystery and intrigue.ith long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair’s face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage",
-                            "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                            "assets/t2v_2.mp4",
-                        ],
-                        [
-                            "A young man with blond hair wearing a yellow jacket stands in a forest and looks around. He has light skin and his hair is styled with a middle part. He looks to the left and then to the right, his gaze lingering in each direction. The camera angle is low, looking up at the man, and remains stationary throughout the video. The background is slightly out of focus, with green trees and the sun shining brightly behind the man. The lighting is natural and warm, with the sun creating a lens flare that moves across the man’s face. The scene is captured in real-life footage.",
-                            "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                            "assets/t2v_1.mp4",
-                        ],
-                        [
-                            "A cyclist races along a winding mountain road. Clad in aerodynamic gear, he pedals intensely, sweat glistening on his brow. The camera alternates between close-ups of his determined expression and wide shots of the breathtaking landscape. Pine trees blur past, and the sky is a crisp blue. The scene is invigorating and competitive.",
-                            "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                            "assets/t2v_0.mp4",
-                        ],
-                    ],
-                    inputs=[txt2vid_prompt, txt2vid_negative_prompt, txt2vid_output],
-                    label="Example Text-to-Video Generations",
-                )
-
-        # Image to Video Tab
-        with gr.TabItem("Image to Video"):
-            with gr.Row():
-                with gr.Column():
-                    img2vid_image = gr.Image(
-                        type="filepath",
-                        label="Step 1: Upload Input Image",
-                        elem_id="image_upload",
-                    )
-                    img2vid_prompt = gr.Textbox(
-                        label="Step 2: Enter Your Prompt",
-                        placeholder="Describe how you want to animate the image (minimum 50 characters)...",
-                        value="A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage.",
-                        lines=5,
-                    )
-                    img2vid_analytics_toggle = Toggle(
-                        label="I agree to share my usage data anonymously to help improve the model features.",
-                        value=True,
-                        interactive=True,
-                    )
-                    img2vid_enhance_toggle = Toggle(
-                        label="Enhance Prompt",
-                        value=False,
-                        interactive=True,
-                    )
-                    img2vid_negative_prompt = gr.Textbox(
-                        label="Step 3: Enter Negative Prompt",
-                        placeholder="Describe what you don't want in the video...",
-                        value="low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                        lines=2,
-                    )
-
-                    img2vid_preset = gr.Dropdown(
-                        choices=[p["label"] for p in preset_options],
-                        value="768x512, 97 frames",
-                        label="Step 3.1: Choose Resolution Preset",
-                    )
-
-                    img2vid_frame_rate = gr.Slider(
-                        label="Step 3.2: Frame Rate",
-                        minimum=21,
-                        maximum=30,
-                        step=1,
-                        value=25,
-                    )
-
-                    img2vid_advanced = create_advanced_options()
-                    img2vid_generate = gr.Button("Step 6: Generate Video", variant="primary", size="lg")
-
-                with gr.Column():
-                    img2vid_output = gr.Video(label="Generated Output")
-
-            with gr.Row():
-                gr.Examples(
-                    examples=[
-                        [
-                            "assets/i2v_i2.png",
-                            "A woman stirs a pot of boiling water on a white electric burner. Her hands, with purple nail polish, hold a wooden spoon and move it in a circular motion within a white pot filled with bubbling water. The pot sits on a white electric burner with black buttons and a digital display. The burner is positioned on a white countertop with a red and white checkered cloth partially visible in the bottom right corner. The camera angle is a direct overhead shot, remaining stationary throughout the scene. The lighting is bright and even, illuminating the scene with a neutral white light. The scene is real-life footage.",
-                            "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                            "assets/i2v_2.mp4",
-                        ],
-                        [
-                            "assets/i2v_i0.png",
-                            "A woman in a long, flowing dress stands in a field, her back to the camera, gazing towards the horizon; her hair is long and light, cascading down her back; she stands beneath the sprawling branches of a large oak tree;  to her left, a classic American car is parked on the dry grass; in the distance, a wrecked car lies on its side; the sky above is a dramatic canvas of bright white clouds against a darker sky; the entire image is in black and white, emphasizing the contrast of light and shadow. The woman is walking slowly towards the car.",
-                            "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                            "assets/i2v_0.mp4",
-                        ],
-                        [
-                            "assets/i2v_i1.png",
-                            "A pair of hands shapes a piece of clay on a pottery wheel, gradually forming a cone shape. The hands, belonging to a person out of frame, are covered in clay and gently press a ball of clay onto the center of a spinning pottery wheel. The hands move in a circular motion, gradually forming a cone shape at the top of the clay. The camera is positioned directly above the pottery wheel, providing a bird’s-eye view of the clay being shaped. The lighting is bright and even, illuminating the clay and the hands working on it. The scene is captured in real-life footage.",
-                            "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly",
-                            "assets/i2v_1.mp4",
-                        ],
-                    ],
-                    inputs=[
-                        img2vid_image,
-                        img2vid_prompt,
-                        img2vid_negative_prompt,
-                        img2vid_output,
-                    ],
-                    label="Example Image-to-Video Generations",
-                )
-
-    # [Previous event handlers remain the same]
-    txt2vid_preset.change(fn=preset_changed, inputs=[txt2vid_preset], outputs=txt2vid_advanced[3:])
-
-    txt2vid_generate.click(
-        fn=generate_video_from_text,
-        inputs=[
-            txt2vid_prompt,
-            txt2vid_enhance_toggle,
-            txt2vid_analytics_toggle,
-            txt2vid_negative_prompt,
-            txt2vid_frame_rate,
-            *txt2vid_advanced,
-        ],
-        outputs=txt2vid_output,
-        concurrency_limit=1,
-        concurrency_id="generate_video",
+        print(f"Error saving video with macro_block_size=1: {e}")
+        try:
+            with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], format='FFMPEG', codec='libx264', quality=8) as video_writer:
+                 for frame_idx in range(video_np.shape[0]):
+                    progress(frame_idx / video_np.shape[0], desc="Saving video (fallback ffmpeg)")
+                    video_writer.append_data(video_np[frame_idx])
+        except Exception as e2:
+            print(f"Fallback video saving error: {e2}")
+            raise gr.Error(f"Failed to save video: {e2}")
+            
+    return output_video_path, seed_ui
+
+def update_task_image():
+    return "image-to-video"
+
+def update_task_text():
+    return "text-to-video"
+
+def update_task_video():
+    return "video-to-video"
+
+# --- Gradio UI Definition ---
+css="""
+#col-container {
+    margin: 0 auto;
+    max-width: 900px;
+}
+"""
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# LTX Video 0.9.8 13B Distilled")
+    gr.Markdown("Fast high quality video generation.**Update (17/07):** now with the new v0.9.8 for improved prompt understanding and detail generation" )
+    gr.Markdown("[Model](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-13b-0.9.8-distilled.safetensors) [GitHub](https://github.com/Lightricks/LTX-Video) [Diffusers](https://huggingface.co/Lightricks/LTX-Video-0.9.8-13B-distilled#diffusers-🧨)")
+    with gr.Row():
+        with gr.Column():
+            with gr.Tab("image-to-video") as image_tab:
+                video_i_hidden = gr.Textbox(label="video_i", visible=False, value=None)
+                image_i2v = gr.Image(label="Input Image", type="filepath", sources=["upload", "webcam", "clipboard"])
+                i2v_prompt = gr.Textbox(label="Prompt", value="The creature from the image starts to move", lines=3)
+                i2v_button = gr.Button("Generate Image-to-Video", variant="primary")
+            with gr.Tab("text-to-video") as text_tab:
+                image_n_hidden = gr.Textbox(label="image_n", visible=False, value=None)
+                video_n_hidden = gr.Textbox(label="video_n", visible=False, value=None)
+                t2v_prompt = gr.Textbox(label="Prompt", value="A majestic dragon flying over a medieval castle", lines=3)
+                t2v_button = gr.Button("Generate Text-to-Video", variant="primary")
+            with gr.Tab("video-to-video", visible=False) as video_tab:
+                image_v_hidden = gr.Textbox(label="image_v", visible=False, value=None)
+                video_v2v = gr.Video(label="Input Video", sources=["upload", "webcam"]) # type defaults to filepath
+                frames_to_use = gr.Slider(label="Frames to use from input video", minimum=9, maximum=MAX_NUM_FRAMES, value=9, step=8, info="Number of initial frames to use for conditioning/transformation. Must be N*8+1.")
+                v2v_prompt = gr.Textbox(label="Prompt", value="Change the style to cinematic anime", lines=3)
+                v2v_button = gr.Button("Generate Video-to-Video", variant="primary")
+
+            duration_input = gr.Slider(
+                label="Video Duration (seconds)", 
+                minimum=0.3, 
+                maximum=8.5, 
+                value=2,  
+                step=0.1, 
+                info=f"Target video duration (0.3s to 8.5s)"
+            )
+            improve_texture = gr.Checkbox(label="Improve Texture (multi-scale)", value=True,visible=False, info="Uses a two-pass generation for better quality, but is slower. Recommended for final output.")
+
+        with gr.Column():
+            output_video = gr.Video(label="Generated Video", interactive=False)
+            # gr.DeepLinkButton()
+
+    with gr.Accordion("Advanced settings", open=False):
+        mode = gr.Dropdown(["text-to-video", "image-to-video", "video-to-video"], label="task", value="image-to-video", visible=False)
+        negative_prompt_input = gr.Textbox(label="Negative Prompt", value="worst quality, inconsistent motion, blurry, jittery, distorted", lines=2)
+        with gr.Row():
+            seed_input = gr.Number(label="Seed", value=42, precision=0, minimum=0, maximum=2**32-1)
+            randomize_seed_input = gr.Checkbox(label="Randomize Seed", value=True)
+        with gr.Row(visible=False):
+            guidance_scale_input = gr.Slider(label="Guidance Scale (CFG)", minimum=1.0, maximum=10.0, value=PIPELINE_CONFIG_YAML.get("first_pass", {}).get("guidance_scale", 1.0), step=0.1, info="Controls how much the prompt influences the output. Higher values = stronger influence.")
+        with gr.Row():
+            height_input = gr.Slider(label="Height", value=512, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
+            width_input = gr.Slider(label="Width", value=704, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
+
+
+    # --- Event handlers for updating dimensions on upload ---
+    def handle_image_upload_for_dims(image_filepath, current_h, current_w):
+        if not image_filepath:  # Image cleared or no image initially
+            # Keep current slider values if image is cleared or no input
+            return gr.update(value=current_h), gr.update(value=current_w)
+        try:
+            img = Image.open(image_filepath)
+            orig_w, orig_h = img.size
+            new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
+            return gr.update(value=new_h), gr.update(value=new_w)
+        except Exception as e:
+            print(f"Error processing image for dimension update: {e}")
+            # Keep current slider values on error
+            return gr.update(value=current_h), gr.update(value=current_w)
+
+    def handle_video_upload_for_dims(video_filepath, current_h, current_w):
+        if not video_filepath:  # Video cleared or no video initially
+            return gr.update(value=current_h), gr.update(value=current_w)
+        try:
+            # Ensure video_filepath is a string for os.path.exists and imageio
+            video_filepath_str = str(video_filepath) 
+            if not os.path.exists(video_filepath_str):
+                print(f"Video file path does not exist for dimension update: {video_filepath_str}")
+                return gr.update(value=current_h), gr.update(value=current_w)
+
+            orig_w, orig_h = -1, -1
+            with imageio.get_reader(video_filepath_str) as reader:
+                meta = reader.get_meta_data()
+                if 'size' in meta:
+                    orig_w, orig_h = meta['size']
+                else:
+                    # Fallback: read first frame if 'size' not in metadata
+                    try:
+                        first_frame = reader.get_data(0)
+                        # Shape is (h, w, c) for frames
+                        orig_h, orig_w = first_frame.shape[0], first_frame.shape[1]
+                    except Exception as e_frame:
+                        print(f"Could not get video size from metadata or first frame: {e_frame}")
+                        return gr.update(value=current_h), gr.update(value=current_w)
+            
+            if orig_w == -1 or orig_h == -1: # If dimensions couldn't be determined
+                 print(f"Could not determine dimensions for video: {video_filepath_str}")
+                 return gr.update(value=current_h), gr.update(value=current_w)
+
+            new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
+            return gr.update(value=new_h), gr.update(value=new_w)
+        except Exception as e:
+            # Log type of video_filepath for debugging if it's not a path-like string
+            print(f"Error processing video for dimension update: {e} (Path: {video_filepath}, Type: {type(video_filepath)})")
+            return gr.update(value=current_h), gr.update(value=current_w)
+
+    
+    image_i2v.upload(
+        fn=handle_image_upload_for_dims,
+        inputs=[image_i2v, height_input, width_input],
+        outputs=[height_input, width_input]
+    )
+    video_v2v.upload(
+        fn=handle_video_upload_for_dims,
+        inputs=[video_v2v, height_input, width_input],
+        outputs=[height_input, width_input]
     )
 
-    img2vid_preset.change(fn=preset_changed, inputs=[img2vid_preset], outputs=img2vid_advanced[3:])
-
-    img2vid_generate.click(
-        fn=generate_video_from_image,
-        inputs=[
-            img2vid_image,
-            img2vid_prompt,
-            img2vid_enhance_toggle,
-            img2vid_analytics_toggle,
-            img2vid_negative_prompt,
-            img2vid_frame_rate,
-            *img2vid_advanced,
-        ],
-        outputs=img2vid_output,
-        concurrency_limit=1,
-        concurrency_id="generate_video",
+    image_tab.select(
+        fn=update_task_image,
+        outputs=[mode]
+    )
+    text_tab.select(
+        fn=update_task_text,
+        outputs=[mode]
     )
+    
+    t2v_inputs = [t2v_prompt, negative_prompt_input, image_n_hidden, video_n_hidden,
+                  height_input, width_input, mode,
+                  duration_input, frames_to_use, 
+                  seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
+    
+    i2v_inputs = [i2v_prompt, negative_prompt_input, image_i2v, video_i_hidden,
+                  height_input, width_input, mode,
+                  duration_input, frames_to_use, 
+                  seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
+
+    v2v_inputs = [v2v_prompt, negative_prompt_input, image_v_hidden, video_v2v,
+                  height_input, width_input, mode,
+                  duration_input, frames_to_use, 
+                  seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
+
+    t2v_button.click(fn=generate, inputs=t2v_inputs, outputs=[output_video, seed_input], api_name="text_to_video")
+    i2v_button.click(fn=generate, inputs=i2v_inputs, outputs=[output_video, seed_input], api_name="image_to_video")
+    v2v_button.click(fn=generate, inputs=v2v_inputs, outputs=[output_video, seed_input], api_name="video_to_video")
 
 if __name__ == "__main__":
-    iface.queue(max_size=64, default_concurrency_limit=1, api_open=False).launch(share=True, show_api=False)
+    if os.path.exists(models_dir) and os.path.isdir(models_dir):
+        print(f"Model directory: {Path(models_dir).resolve()}")
+    
+    demo.queue().launch(debug=True, share=False, mcp_server=True)

inference.py

@@ -1,162 +1,206 @@
-import torch
-from xora.models.autoencoders.causal_video_autoencoder import CausalVideoAutoencoder
-from xora.models.transformers.transformer3d import Transformer3DModel
-from xora.models.transformers.symmetric_patchifier import SymmetricPatchifier
-from xora.schedulers.rf import RectifiedFlowScheduler
-from xora.pipelines.pipeline_xora_video import XoraVideoPipeline
-from pathlib import Path
-from transformers import T5EncoderModel, T5Tokenizer
-import safetensors.torch
-import json
 import argparse
-from xora.utils.conditioning_method import ConditioningMethod
 import os
+import random
+from datetime import datetime
+from pathlib import Path
+from diffusers.utils import logging
+from typing import Optional, List, Union
+import yaml
+
+import imageio
+import json
 import numpy as np
+import torch
 import cv2
+from safetensors import safe_open
 from PIL import Image
-import random
-
-RECOMMENDED_RESOLUTIONS = [
-    (704, 1216, 41),
-    (704, 1088, 49),
-    (640, 1056, 57),
-    (608, 992, 65),
-    (608, 896, 73),
-    (544, 896, 81),
-    (544, 832, 89),
-    (512, 800, 97),
-    (512, 768, 97),
-    (480, 800, 105),
-    (480, 736, 113),
-    (480, 704, 121),
-    (448, 704, 129),
-    (448, 672, 137),
-    (416, 640, 153),
-    (384, 672, 161),
-    (384, 640, 169),
-    (384, 608, 177),
-    (384, 576, 185),
-    (352, 608, 193),
-    (352, 576, 201),
-    (352, 544, 209),
-    (352, 512, 225),
-    (352, 512, 233),
-    (320, 544, 241),
-    (320, 512, 249),
-    (320, 512, 257),
-]
-
-
-def load_vae(vae_dir):
-    vae_ckpt_path = vae_dir / "vae_diffusion_pytorch_model.safetensors"
-    vae_config_path = vae_dir / "config.json"
-    with open(vae_config_path, "r") as f:
-        vae_config = json.load(f)
-    vae = CausalVideoAutoencoder.from_config(vae_config)
-    vae_state_dict = safetensors.torch.load_file(vae_ckpt_path)
-    vae.load_state_dict(vae_state_dict)
+from transformers import (
+    T5EncoderModel,
+    T5Tokenizer,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    AutoTokenizer,
+)
+from huggingface_hub import hf_hub_download
+
+from ltx_video.models.autoencoders.causal_video_autoencoder import (
+    CausalVideoAutoencoder,
+)
+from ltx_video.models.transformers.symmetric_patchifier import SymmetricPatchifier
+from ltx_video.models.transformers.transformer3d import Transformer3DModel
+from ltx_video.pipelines.pipeline_ltx_video import (
+    ConditioningItem,
+    LTXVideoPipeline,
+    LTXMultiScalePipeline,
+)
+from ltx_video.schedulers.rf import RectifiedFlowScheduler
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
+from ltx_video.models.autoencoders.latent_upsampler import LatentUpsampler
+import ltx_video.pipelines.crf_compressor as crf_compressor
+
+MAX_HEIGHT = 720
+MAX_WIDTH = 1280
+MAX_NUM_FRAMES = 257
+
+logger = logging.get_logger("LTX-Video")
+
+
+def get_total_gpu_memory():
     if torch.cuda.is_available():
-        vae = vae.cuda()
-    return vae.to(torch.bfloat16)
+        total_memory = torch.cuda.get_device_properties(0).total_memory / (1024**3)
+        return total_memory
+    return 0
 
 
-def load_unet(unet_dir):
-    unet_ckpt_path = unet_dir / "unet_diffusion_pytorch_model.safetensors"
-    unet_config_path = unet_dir / "config.json"
-    transformer_config = Transformer3DModel.load_config(unet_config_path)
-    transformer = Transformer3DModel.from_config(transformer_config)
-    unet_state_dict = safetensors.torch.load_file(unet_ckpt_path)
-    transformer.load_state_dict(unet_state_dict, strict=True)
+def get_device():
     if torch.cuda.is_available():
-        transformer = transformer.cuda()
-    return transformer
-
-
-def load_scheduler(scheduler_dir):
-    scheduler_config_path = scheduler_dir / "scheduler_config.json"
-    scheduler_config = RectifiedFlowScheduler.load_config(scheduler_config_path)
-    return RectifiedFlowScheduler.from_config(scheduler_config)
-
+        return "cuda"
+    elif torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+def load_image_to_tensor_with_resize_and_crop(
+    image_input: Union[str, Image.Image],
+    target_height: int = 512,
+    target_width: int = 768,
+    just_crop: bool = False,
+) -> torch.Tensor:
+    """Load and process an image into a tensor.
+
+    Args:
+        image_input: Either a file path (str) or a PIL Image object
+        target_height: Desired height of output tensor
+        target_width: Desired width of output tensor
+        just_crop: If True, only crop the image to the target size without resizing
+    """
+    if isinstance(image_input, str):
+        image = Image.open(image_input).convert("RGB")
+    elif isinstance(image_input, Image.Image):
+        image = image_input
+    else:
+        raise ValueError("image_input must be either a file path or a PIL Image object")
 
-def center_crop_and_resize(frame, target_height, target_width):
-    h, w, _ = frame.shape
+    input_width, input_height = image.size
     aspect_ratio_target = target_width / target_height
-    aspect_ratio_frame = w / h
+    aspect_ratio_frame = input_width / input_height
     if aspect_ratio_frame > aspect_ratio_target:
-        new_width = int(h * aspect_ratio_target)
-        x_start = (w - new_width) // 2
-        frame_cropped = frame[:, x_start : x_start + new_width]
+        new_width = int(input_height * aspect_ratio_target)
+        new_height = input_height
+        x_start = (input_width - new_width) // 2
+        y_start = 0
     else:
-        new_height = int(w / aspect_ratio_target)
-        y_start = (h - new_height) // 2
-        frame_cropped = frame[y_start : y_start + new_height, :]
-    frame_resized = cv2.resize(frame_cropped, (target_width, target_height))
-    return frame_resized
-
-
-def load_video_to_tensor_with_resize(video_path, target_height, target_width):
-    cap = cv2.VideoCapture(video_path)
-    frames = []
-    while True:
-        ret, frame = cap.read()
-        if not ret:
-            break
-        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-        if target_height is not None:
-            frame_resized = center_crop_and_resize(
-                frame_rgb, target_height, target_width
-            )
-        else:
-            frame_resized = frame_rgb
-        frames.append(frame_resized)
-    cap.release()
-    video_np = (np.array(frames) / 127.5) - 1.0
-    video_tensor = torch.tensor(video_np).permute(3, 0, 1, 2).float()
-    return video_tensor
-
-
-def load_image_to_tensor_with_resize(image_path, target_height=512, target_width=768):
-    image = Image.open(image_path).convert("RGB")
-    image_np = np.array(image)
-    frame_resized = center_crop_and_resize(image_np, target_height, target_width)
-    frame_tensor = torch.tensor(frame_resized).permute(2, 0, 1).float()
+        new_width = input_width
+        new_height = int(input_width / aspect_ratio_target)
+        x_start = 0
+        y_start = (input_height - new_height) // 2
+
+    image = image.crop((x_start, y_start, x_start + new_width, y_start + new_height))
+    if not just_crop:
+        image = image.resize((target_width, target_height))
+
+    image = np.array(image)
+    image = cv2.GaussianBlur(image, (3, 3), 0)
+    frame_tensor = torch.from_numpy(image).float()
+    frame_tensor = crf_compressor.compress(frame_tensor / 255.0) * 255.0
+    frame_tensor = frame_tensor.permute(2, 0, 1)
     frame_tensor = (frame_tensor / 127.5) - 1.0
     # Create 5D tensor: (batch_size=1, channels=3, num_frames=1, height, width)
     return frame_tensor.unsqueeze(0).unsqueeze(2)
 
 
+def calculate_padding(
+    source_height: int, source_width: int, target_height: int, target_width: int
+) -> tuple[int, int, int, int]:
+
+    # Calculate total padding needed
+    pad_height = target_height - source_height
+    pad_width = target_width - source_width
+
+    # Calculate padding for each side
+    pad_top = pad_height // 2
+    pad_bottom = pad_height - pad_top  # Handles odd padding
+    pad_left = pad_width // 2
+    pad_right = pad_width - pad_left  # Handles odd padding
+
+    # Return padded tensor
+    # Padding format is (left, right, top, bottom)
+    padding = (pad_left, pad_right, pad_top, pad_bottom)
+    return padding
+
+
+def convert_prompt_to_filename(text: str, max_len: int = 20) -> str:
+    # Remove non-letters and convert to lowercase
+    clean_text = "".join(
+        char.lower() for char in text if char.isalpha() or char.isspace()
+    )
+
+    # Split into words
+    words = clean_text.split()
+
+    # Build result string keeping track of length
+    result = []
+    current_length = 0
+
+    for word in words:
+        # Add word length plus 1 for underscore (except for first word)
+        new_length = current_length + len(word)
+
+        if new_length <= max_len:
+            result.append(word)
+            current_length += len(word)
+        else:
+            break
+
+    return "-".join(result)
+
+
+# Generate output video name
+def get_unique_filename(
+    base: str,
+    ext: str,
+    prompt: str,
+    seed: int,
+    resolution: tuple[int, int, int],
+    dir: Path,
+    endswith=None,
+    index_range=1000,
+) -> Path:
+    base_filename = f"{base}_{convert_prompt_to_filename(prompt, max_len=30)}_{seed}_{resolution[0]}x{resolution[1]}x{resolution[2]}"
+    for i in range(index_range):
+        filename = dir / f"{base_filename}_{i}{endswith if endswith else ''}{ext}"
+        if not os.path.exists(filename):
+            return filename
+    raise FileExistsError(
+        f"Could not find a unique filename after {index_range} attempts."
+    )
+
+
+def seed_everething(seed: int):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed(seed)
+    if torch.backends.mps.is_available():
+        torch.mps.manual_seed(seed)
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Load models from separate directories and run the pipeline."
     )
 
     # Directories
-    parser.add_argument(
-        "--ckpt_dir",
-        type=str,
-        required=True,
-        help="Path to the directory containing unet, vae, and scheduler subdirectories",
-    )
-    parser.add_argument(
-        "--input_video_path",
-        type=str,
-        help="Path to the input video file (first frame used)",
-    )
-    parser.add_argument(
-        "--input_image_path", type=str, help="Path to the input image file"
-    )
     parser.add_argument(
         "--output_path",
         type=str,
         default=None,
-        help="Path to save output video, if None will save in working directory.",
+        help="Path to the folder to save output video, if None will save in outputs/ directory.",
     )
     parser.add_argument("--seed", type=int, default="171198")
 
     # Pipeline parameters
-    parser.add_argument(
-        "--num_inference_steps", type=int, default=40, help="Number of inference steps"
-    )
     parser.add_argument(
         "--num_images_per_prompt",
         type=int,
@@ -164,21 +208,21 @@ def main():
         help="Number of images per prompt",
     )
     parser.add_argument(
-        "--guidance_scale",
+        "--image_cond_noise_scale",
         type=float,
-        default=3,
-        help="Guidance scale for the pipeline",
+        default=0.15,
+        help="Amount of noise to add to the conditioned image",
     )
     parser.add_argument(
         "--height",
         type=int,
-        default=None,
+        default=704,
         help="Height of the output video frames. Optional if an input image provided.",
     )
     parser.add_argument(
         "--width",
         type=int,
-        default=None,
+        default=1216,
         help="Width of the output video frames. If None will infer from input image.",
     )
     parser.add_argument(
@@ -188,13 +232,18 @@ def main():
         help="Number of frames to generate in the output video",
     )
     parser.add_argument(
-        "--frame_rate", type=int, default=25, help="Frame rate for the output video"
+        "--frame_rate", type=int, default=30, help="Frame rate for the output video"
     )
-
     parser.add_argument(
-        "--bfloat16",
-        action="store_true",
-        help="Denoise in bfloat16",
+        "--device",
+        default=None,
+        help="Device to run inference on. If not specified, will automatically detect and use CUDA or MPS if available, else CPU.",
+    )
+    parser.add_argument(
+        "--pipeline_config",
+        type=str,
+        default="configs/ltxv-13b-0.9.7-dev.yaml",
+        help="The path to the config file for the pipeline, which contains the parameters for the pipeline",
     )
 
     # Prompts
@@ -209,161 +258,517 @@ def main():
         default="worst quality, inconsistent motion, blurry, jittery, distorted",
         help="Negative prompt for undesired features",
     )
+
     parser.add_argument(
-        "--custom_resolution",
+        "--offload_to_cpu",
         action="store_true",
-        default=False,
-        help="Enable custom resolution (not in recommneded resolutions) if specified (default: False)",
+        help="Offloading unnecessary computations to CPU.",
     )
 
-    args = parser.parse_args()
+    # video-to-video arguments:
+    parser.add_argument(
+        "--input_media_path",
+        type=str,
+        default=None,
+        help="Path to the input video (or imaage) to be modified using the video-to-video pipeline",
+    )
 
-    if args.input_image_path is None and args.input_video_path is None:
-        assert (
-            args.height is not None and args.width is not None
-        ), "Must enter height and width for text to image generation."
-
-    # Load media (video or image)
-    if args.input_video_path:
-        media_items = load_video_to_tensor_with_resize(
-            args.input_video_path, args.height, args.width
-        ).unsqueeze(0)
-    elif args.input_image_path:
-        media_items = load_image_to_tensor_with_resize(
-            args.input_image_path, args.height, args.width
-        )
+    # Conditioning arguments
+    parser.add_argument(
+        "--conditioning_media_paths",
+        type=str,
+        nargs="*",
+        help="List of paths to conditioning media (images or videos). Each path will be used as a conditioning item.",
+    )
+    parser.add_argument(
+        "--conditioning_strengths",
+        type=float,
+        nargs="*",
+        help="List of conditioning strengths (between 0 and 1) for each conditioning item. Must match the number of conditioning items.",
+    )
+    parser.add_argument(
+        "--conditioning_start_frames",
+        type=int,
+        nargs="*",
+        help="List of frame indices where each conditioning item should be applied. Must match the number of conditioning items.",
+    )
+
+    args = parser.parse_args()
+    logger.warning(f"Running generation with arguments: {args}")
+    infer(**vars(args))
+
+
+def create_ltx_video_pipeline(
+    ckpt_path: str,
+    precision: str,
+    text_encoder_model_name_or_path: str,
+    sampler: Optional[str] = None,
+    device: Optional[str] = None,
+    enhance_prompt: bool = False,
+    prompt_enhancer_image_caption_model_name_or_path: Optional[str] = None,
+    prompt_enhancer_llm_model_name_or_path: Optional[str] = None,
+) -> LTXVideoPipeline:
+    ckpt_path = Path(ckpt_path)
+    assert os.path.exists(
+        ckpt_path
+    ), f"Ckpt path provided (--ckpt_path) {ckpt_path} does not exist"
+
+    with safe_open(ckpt_path, framework="pt") as f:
+        metadata = f.metadata()
+        config_str = metadata.get("config")
+        configs = json.loads(config_str)
+        allowed_inference_steps = configs.get("allowed_inference_steps", None)
+
+    vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)
+    transformer = Transformer3DModel.from_pretrained(ckpt_path)
+
+    # Use constructor if sampler is specified, otherwise use from_pretrained
+    if sampler == "from_checkpoint" or not sampler:
+        scheduler = RectifiedFlowScheduler.from_pretrained(ckpt_path)
     else:
-        media_items = None
-
-    height = args.height if args.height else media_items.shape[-2]
-    width = args.width if args.width else media_items.shape[-1]
-    assert height % 32 == 0, f"Height ({height}) should be divisible by 32."
-    assert width % 32 == 0, f"Width ({width}) should be divisible by 32."
-    assert (
-        height,
-        width,
-        args.num_frames,
-    ) in RECOMMENDED_RESOLUTIONS or args.custom_resolution, f"The selected resolution + num frames combination is not supported, results would be suboptimal. Supported (h,w,f) are: {RECOMMENDED_RESOLUTIONS}. Use --custom_resolution to enable working with this resolution."
-
-    # Paths for the separate mode directories
-    ckpt_dir = Path(args.ckpt_dir)
-    unet_dir = ckpt_dir / "unet"
-    vae_dir = ckpt_dir / "vae"
-    scheduler_dir = ckpt_dir / "scheduler"
-
-    # Load models
-    vae = load_vae(vae_dir)
-    unet = load_unet(unet_dir)
-    scheduler = load_scheduler(scheduler_dir)
-    patchifier = SymmetricPatchifier(patch_size=1)
+        scheduler = RectifiedFlowScheduler(
+            sampler=("Uniform" if sampler.lower() == "uniform" else "LinearQuadratic")
+        )
+
     text_encoder = T5EncoderModel.from_pretrained(
-        "PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="text_encoder"
+        text_encoder_model_name_or_path, subfolder="text_encoder"
     )
-    if torch.cuda.is_available():
-        text_encoder = text_encoder.to("cuda")
+    patchifier = SymmetricPatchifier(patch_size=1)
     tokenizer = T5Tokenizer.from_pretrained(
-        "PixArt-alpha/PixArt-XL-2-1024-MS", subfolder="tokenizer"
+        text_encoder_model_name_or_path, subfolder="tokenizer"
     )
 
-    if args.bfloat16 and unet.dtype != torch.bfloat16:
-        unet = unet.to(torch.bfloat16)
+    transformer = transformer.to(device)
+    vae = vae.to(device)
+    text_encoder = text_encoder.to(device)
+
+    if enhance_prompt:
+        prompt_enhancer_image_caption_model = AutoModelForCausalLM.from_pretrained(
+            prompt_enhancer_image_caption_model_name_or_path, trust_remote_code=True
+        )
+        prompt_enhancer_image_caption_processor = AutoProcessor.from_pretrained(
+            prompt_enhancer_image_caption_model_name_or_path, trust_remote_code=True
+        )
+        prompt_enhancer_llm_model = AutoModelForCausalLM.from_pretrained(
+            prompt_enhancer_llm_model_name_or_path,
+            torch_dtype="bfloat16",
+        )
+        prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained(
+            prompt_enhancer_llm_model_name_or_path,
+        )
+    else:
+        prompt_enhancer_image_caption_model = None
+        prompt_enhancer_image_caption_processor = None
+        prompt_enhancer_llm_model = None
+        prompt_enhancer_llm_tokenizer = None
+
+    vae = vae.to(torch.bfloat16)
+    if precision == "bfloat16" and transformer.dtype != torch.bfloat16:
+        transformer = transformer.to(torch.bfloat16)
+    text_encoder = text_encoder.to(torch.bfloat16)
 
     # Use submodels for the pipeline
     submodel_dict = {
-        "transformer": unet,
+        "transformer": transformer,
         "patchifier": patchifier,
         "text_encoder": text_encoder,
         "tokenizer": tokenizer,
         "scheduler": scheduler,
         "vae": vae,
+        "prompt_enhancer_image_caption_model": prompt_enhancer_image_caption_model,
+        "prompt_enhancer_image_caption_processor": prompt_enhancer_image_caption_processor,
+        "prompt_enhancer_llm_model": prompt_enhancer_llm_model,
+        "prompt_enhancer_llm_tokenizer": prompt_enhancer_llm_tokenizer,
+        "allowed_inference_steps": allowed_inference_steps,
     }
 
-    pipeline = XoraVideoPipeline(**submodel_dict)
-    if torch.cuda.is_available():
-        pipeline = pipeline.to("cuda")
+    pipeline = LTXVideoPipeline(**submodel_dict)
+    pipeline = pipeline.to(device)
+    return pipeline
+
+
+def create_latent_upsampler(latent_upsampler_model_path: str, device: str):
+    latent_upsampler = LatentUpsampler.from_pretrained(latent_upsampler_model_path)
+    latent_upsampler.to(device)
+    latent_upsampler.eval()
+    return latent_upsampler
+
+
+def infer(
+    output_path: Optional[str],
+    seed: int,
+    pipeline_config: str,
+    image_cond_noise_scale: float,
+    height: Optional[int],
+    width: Optional[int],
+    num_frames: int,
+    frame_rate: int,
+    prompt: str,
+    negative_prompt: str,
+    offload_to_cpu: bool,
+    input_media_path: Optional[str] = None,
+    conditioning_media_paths: Optional[List[str]] = None,
+    conditioning_strengths: Optional[List[float]] = None,
+    conditioning_start_frames: Optional[List[int]] = None,
+    device: Optional[str] = None,
+    **kwargs,
+):
+    # check if pipeline_config is a file
+    if not os.path.isfile(pipeline_config):
+        raise ValueError(f"Pipeline config file {pipeline_config} does not exist")
+    with open(pipeline_config, "r") as f:
+        pipeline_config = yaml.safe_load(f)
+
+    models_dir = "MODEL_DIR"
+
+    ltxv_model_name_or_path = pipeline_config["checkpoint_path"]
+    if not os.path.isfile(ltxv_model_name_or_path):
+        ltxv_model_path = hf_hub_download(
+            repo_id="Lightricks/LTX-Video",
+            filename=ltxv_model_name_or_path,
+            local_dir=models_dir,
+            repo_type="model",
+        )
+    else:
+        ltxv_model_path = ltxv_model_name_or_path
+
+    spatial_upscaler_model_name_or_path = pipeline_config.get(
+        "spatial_upscaler_model_path"
+    )
+    if spatial_upscaler_model_name_or_path and not os.path.isfile(
+        spatial_upscaler_model_name_or_path
+    ):
+        spatial_upscaler_model_path = hf_hub_download(
+            repo_id="Lightricks/LTX-Video",
+            filename=spatial_upscaler_model_name_or_path,
+            local_dir=models_dir,
+            repo_type="model",
+        )
+    else:
+        spatial_upscaler_model_path = spatial_upscaler_model_name_or_path
+
+    if kwargs.get("input_image_path", None):
+        logger.warning(
+            "Please use conditioning_media_paths instead of input_image_path."
+        )
+        assert not conditioning_media_paths and not conditioning_start_frames
+        conditioning_media_paths = [kwargs["input_image_path"]]
+        conditioning_start_frames = [0]
+
+    # Validate conditioning arguments
+    if conditioning_media_paths:
+        # Use default strengths of 1.0
+        if not conditioning_strengths:
+            conditioning_strengths = [1.0] * len(conditioning_media_paths)
+        if not conditioning_start_frames:
+            raise ValueError(
+                "If `conditioning_media_paths` is provided, "
+                "`conditioning_start_frames` must also be provided"
+            )
+        if len(conditioning_media_paths) != len(conditioning_strengths) or len(
+            conditioning_media_paths
+        ) != len(conditioning_start_frames):
+            raise ValueError(
+                "`conditioning_media_paths`, `conditioning_strengths`, "
+                "and `conditioning_start_frames` must have the same length"
+            )
+        if any(s < 0 or s > 1 for s in conditioning_strengths):
+            raise ValueError("All conditioning strengths must be between 0 and 1")
+        if any(f < 0 or f >= num_frames for f in conditioning_start_frames):
+            raise ValueError(
+                f"All conditioning start frames must be between 0 and {num_frames-1}"
+            )
+
+    seed_everething(seed)
+    if offload_to_cpu and not torch.cuda.is_available():
+        logger.warning(
+            "offload_to_cpu is set to True, but offloading will not occur since the model is already running on CPU."
+        )
+        offload_to_cpu = False
+    else:
+        offload_to_cpu = offload_to_cpu and get_total_gpu_memory() < 30
+
+    output_dir = (
+        Path(output_path)
+        if output_path
+        else Path(f"outputs/{datetime.today().strftime('%Y-%m-%d')}")
+    )
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Adjust dimensions to be divisible by 32 and num_frames to be (N * 8 + 1)
+    height_padded = ((height - 1) // 32 + 1) * 32
+    width_padded = ((width - 1) // 32 + 1) * 32
+    num_frames_padded = ((num_frames - 2) // 8 + 1) * 8 + 1
+
+    padding = calculate_padding(height, width, height_padded, width_padded)
+
+    logger.warning(
+        f"Padded dimensions: {height_padded}x{width_padded}x{num_frames_padded}"
+    )
+
+    prompt_enhancement_words_threshold = pipeline_config[
+        "prompt_enhancement_words_threshold"
+    ]
+
+    prompt_word_count = len(prompt.split())
+    enhance_prompt = (
+        prompt_enhancement_words_threshold > 0
+        and prompt_word_count < prompt_enhancement_words_threshold
+    )
+
+    if prompt_enhancement_words_threshold > 0 and not enhance_prompt:
+        logger.info(
+            f"Prompt has {prompt_word_count} words, which exceeds the threshold of {prompt_enhancement_words_threshold}. Prompt enhancement disabled."
+        )
+
+    precision = pipeline_config["precision"]
+    text_encoder_model_name_or_path = pipeline_config["text_encoder_model_name_or_path"]
+    sampler = pipeline_config["sampler"]
+    prompt_enhancer_image_caption_model_name_or_path = pipeline_config[
+        "prompt_enhancer_image_caption_model_name_or_path"
+    ]
+    prompt_enhancer_llm_model_name_or_path = pipeline_config[
+        "prompt_enhancer_llm_model_name_or_path"
+    ]
+
+    pipeline = create_ltx_video_pipeline(
+        ckpt_path=ltxv_model_path,
+        precision=precision,
+        text_encoder_model_name_or_path=text_encoder_model_name_or_path,
+        sampler=sampler,
+        device=kwargs.get("device", get_device()),
+        enhance_prompt=enhance_prompt,
+        prompt_enhancer_image_caption_model_name_or_path=prompt_enhancer_image_caption_model_name_or_path,
+        prompt_enhancer_llm_model_name_or_path=prompt_enhancer_llm_model_name_or_path,
+    )
+
+    if pipeline_config.get("pipeline_type", None) == "multi-scale":
+        if not spatial_upscaler_model_path:
+            raise ValueError(
+                "spatial upscaler model path is missing from pipeline config file and is required for multi-scale rendering"
+            )
+        latent_upsampler = create_latent_upsampler(
+            spatial_upscaler_model_path, pipeline.device
+        )
+        pipeline = LTXMultiScalePipeline(pipeline, latent_upsampler=latent_upsampler)
+
+    media_item = None
+    if input_media_path:
+        media_item = load_media_file(
+            media_path=input_media_path,
+            height=height,
+            width=width,
+            max_frames=num_frames_padded,
+            padding=padding,
+        )
+
+    conditioning_items = (
+        prepare_conditioning(
+            conditioning_media_paths=conditioning_media_paths,
+            conditioning_strengths=conditioning_strengths,
+            conditioning_start_frames=conditioning_start_frames,
+            height=height,
+            width=width,
+            num_frames=num_frames,
+            padding=padding,
+            pipeline=pipeline,
+        )
+        if conditioning_media_paths
+        else None
+    )
+
+    stg_mode = pipeline_config.get("stg_mode", "attention_values")
+    del pipeline_config["stg_mode"]
+    if stg_mode.lower() == "stg_av" or stg_mode.lower() == "attention_values":
+        skip_layer_strategy = SkipLayerStrategy.AttentionValues
+    elif stg_mode.lower() == "stg_as" or stg_mode.lower() == "attention_skip":
+        skip_layer_strategy = SkipLayerStrategy.AttentionSkip
+    elif stg_mode.lower() == "stg_r" or stg_mode.lower() == "residual":
+        skip_layer_strategy = SkipLayerStrategy.Residual
+    elif stg_mode.lower() == "stg_t" or stg_mode.lower() == "transformer_block":
+        skip_layer_strategy = SkipLayerStrategy.TransformerBlock
+    else:
+        raise ValueError(f"Invalid spatiotemporal guidance mode: {stg_mode}")
 
     # Prepare input for the pipeline
     sample = {
-        "prompt": args.prompt,
+        "prompt": prompt,
         "prompt_attention_mask": None,
-        "negative_prompt": args.negative_prompt,
+        "negative_prompt": negative_prompt,
         "negative_prompt_attention_mask": None,
-        "media_items": media_items,
     }
 
-    random.seed(args.seed)
-    np.random.seed(args.seed)
-    torch.manual_seed(args.seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed(args.seed)
-
-    generator = torch.Generator(
-        device="cuda" if torch.cuda.is_available() else "cpu"
-    ).manual_seed(args.seed)
+    device = device or get_device()
+    generator = torch.Generator(device=device).manual_seed(seed)
 
     images = pipeline(
-        num_inference_steps=args.num_inference_steps,
-        num_images_per_prompt=args.num_images_per_prompt,
-        guidance_scale=args.guidance_scale,
+        **pipeline_config,
+        skip_layer_strategy=skip_layer_strategy,
         generator=generator,
         output_type="pt",
         callback_on_step_end=None,
-        height=height,
-        width=width,
-        num_frames=args.num_frames,
-        frame_rate=args.frame_rate,
+        height=height_padded,
+        width=width_padded,
+        num_frames=num_frames_padded,
+        frame_rate=frame_rate,
         **sample,
+        media_items=media_item,
+        conditioning_items=conditioning_items,
         is_video=True,
         vae_per_channel_normalize=True,
-        conditioning_method=(
-            ConditioningMethod.FIRST_FRAME
-            if media_items is not None
-            else ConditioningMethod.UNCONDITIONAL
-        ),
-        mixed_precision=not args.bfloat16,
+        image_cond_noise_scale=image_cond_noise_scale,
+        mixed_precision=(precision == "mixed_precision"),
+        offload_to_cpu=offload_to_cpu,
+        device=device,
+        enhance_prompt=enhance_prompt,
     ).images
 
-    # Save output video
-    def get_unique_filename(base, ext, dir=".", index_range=1000):
-        for i in range(index_range):
-            filename = os.path.join(dir, f"{base}_{i}{ext}")
-            if not os.path.exists(filename):
-                return filename
-        raise FileExistsError(
-            f"Could not find a unique filename after {index_range} attempts."
-        )
+    # Crop the padded images to the desired resolution and number of frames
+    (pad_left, pad_right, pad_top, pad_bottom) = padding
+    pad_bottom = -pad_bottom
+    pad_right = -pad_right
+    if pad_bottom == 0:
+        pad_bottom = images.shape[3]
+    if pad_right == 0:
+        pad_right = images.shape[4]
+    images = images[:, :, :num_frames, pad_top:pad_bottom, pad_left:pad_right]
 
     for i in range(images.shape[0]):
         # Gathering from B, C, F, H, W to C, F, H, W and then permuting to F, H, W, C
         video_np = images[i].permute(1, 2, 3, 0).cpu().float().numpy()
         # Unnormalizing images to [0, 255] range
         video_np = (video_np * 255).astype(np.uint8)
-        fps = args.frame_rate
+        fps = frame_rate
         height, width = video_np.shape[1:3]
+        # In case a single image is generated
         if video_np.shape[0] == 1:
-            output_filename = (
-                args.output_path
-                if args.output_path is not None
-                else get_unique_filename(f"image_output_{i}", ".png", ".")
+            output_filename = get_unique_filename(
+                f"image_output_{i}",
+                ".png",
+                prompt=prompt,
+                seed=seed,
+                resolution=(height, width, num_frames),
+                dir=output_dir,
             )
-            cv2.imwrite(
-                output_filename, video_np[0][..., ::-1]
-            )  # Save single frame as image
+            imageio.imwrite(output_filename, video_np[0])
         else:
-            output_filename = (
-                args.output_path
-                if args.output_path is not None
-                else get_unique_filename(f"video_output_{i}", ".mp4", ".")
+            output_filename = get_unique_filename(
+                f"video_output_{i}",
+                ".mp4",
+                prompt=prompt,
+                seed=seed,
+                resolution=(height, width, num_frames),
+                dir=output_dir,
             )
 
-            out = cv2.VideoWriter(
-                output_filename, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)
+            # Write video
+            with imageio.get_writer(output_filename, fps=fps) as video:
+                for frame in video_np:
+                    video.append_data(frame)
+
+        logger.warning(f"Output saved to {output_filename}")
+
+
+def prepare_conditioning(
+    conditioning_media_paths: List[str],
+    conditioning_strengths: List[float],
+    conditioning_start_frames: List[int],
+    height: int,
+    width: int,
+    num_frames: int,
+    padding: tuple[int, int, int, int],
+    pipeline: LTXVideoPipeline,
+) -> Optional[List[ConditioningItem]]:
+    """Prepare conditioning items based on input media paths and their parameters.
+
+    Args:
+        conditioning_media_paths: List of paths to conditioning media (images or videos)
+        conditioning_strengths: List of conditioning strengths for each media item
+        conditioning_start_frames: List of frame indices where each item should be applied
+        height: Height of the output frames
+        width: Width of the output frames
+        num_frames: Number of frames in the output video
+        padding: Padding to apply to the frames
+        pipeline: LTXVideoPipeline object used for condition video trimming
+
+    Returns:
+        A list of ConditioningItem objects.
+    """
+    conditioning_items = []
+    for path, strength, start_frame in zip(
+        conditioning_media_paths, conditioning_strengths, conditioning_start_frames
+    ):
+        num_input_frames = orig_num_input_frames = get_media_num_frames(path)
+        if hasattr(pipeline, "trim_conditioning_sequence") and callable(
+            getattr(pipeline, "trim_conditioning_sequence")
+        ):
+            num_input_frames = pipeline.trim_conditioning_sequence(
+                start_frame, orig_num_input_frames, num_frames
             )
+        if num_input_frames < orig_num_input_frames:
+            logger.warning(
+                f"Trimming conditioning video {path} from {orig_num_input_frames} to {num_input_frames} frames."
+            )
+
+        media_tensor = load_media_file(
+            media_path=path,
+            height=height,
+            width=width,
+            max_frames=num_input_frames,
+            padding=padding,
+            just_crop=True,
+        )
+        conditioning_items.append(ConditioningItem(media_tensor, start_frame, strength))
+    return conditioning_items
+
 
-            for frame in video_np[..., ::-1]:
-                out.write(frame)
-            out.release()
+def get_media_num_frames(media_path: str) -> int:
+    is_video = any(
+        media_path.lower().endswith(ext) for ext in [".mp4", ".avi", ".mov", ".mkv"]
+    )
+    num_frames = 1
+    if is_video:
+        reader = imageio.get_reader(media_path)
+        num_frames = reader.count_frames()
+        reader.close()
+    return num_frames
+
+
+def load_media_file(
+    media_path: str,
+    height: int,
+    width: int,
+    max_frames: int,
+    padding: tuple[int, int, int, int],
+    just_crop: bool = False,
+) -> torch.Tensor:
+    is_video = any(
+        media_path.lower().endswith(ext) for ext in [".mp4", ".avi", ".mov", ".mkv"]
+    )
+    if is_video:
+        reader = imageio.get_reader(media_path)
+        num_input_frames = min(reader.count_frames(), max_frames)
+
+        # Read and preprocess the relevant frames from the video file.
+        frames = []
+        for i in range(num_input_frames):
+            frame = Image.fromarray(reader.get_data(i))
+            frame_tensor = load_image_to_tensor_with_resize_and_crop(
+                frame, height, width, just_crop=just_crop
+            )
+            frame_tensor = torch.nn.functional.pad(frame_tensor, padding)
+            frames.append(frame_tensor)
+        reader.close()
+
+        # Stack frames along the temporal dimension
+        media_tensor = torch.cat(frames, dim=2)
+    else:  # Input image
+        media_tensor = load_image_to_tensor_with_resize_and_crop(
+            media_path, height, width, just_crop=just_crop
+        )
+        media_tensor = torch.nn.functional.pad(media_tensor, padding)
+    return media_tensor
 
 
 if __name__ == "__main__":
-    main()
+    main()

requirements.txt

@@ -1,14 +1,15 @@
-huggingface_hub<=0.25
-torch
-diffusers==0.28.2
-transformers==4.44.2
-sentencepiece>=0.1.96
 accelerate
-einops
-matplotlib
+transformers
+sentencepiece
+pillow
+numpy
+torchvision
+huggingface_hub
+spaces
 opencv-python
-beautifulsoup4
-ftfy
-gradio
-openai
-gradio_toggle
+imageio
+imageio-ffmpeg
+einops
+timm
+av
+git+https://github.com/huggingface/diffusers.git@main