--- license: apache-2.0 base_model: - Qwen/Qwen2.5-7B-Instruct pipeline_tag: any-to-any language: - en - zh --- # Ola-7B ## Model Summary The Ola-7B model is developed by people from Tencent, Tsinghua University and Nanyang Technological University. Based on Qwen2.5 language model, it is trained on text, image, video and audio data with a context window of 32K tokens. It can take both image/video, text and audio as input and output text. Ola offers an on-demand solution to seamlessly and efficiently process visual inputs with arbitrary spatial sizes and temporal lengths. - **Repository:** https://github.com/Ola-Omni/Ola - **Languages:** English, Chinese - **Paper:** https://huggingface.co/papers/2502.04328 ## Use 1. Download the speech encoder at https://huggingface.co/THUdyh/Ola_speech_encoders. 2. Replace the path in config.json with local path of speech encoders. We provide a simple generation process for using our model. For more details, please refer to our [Github Repo](https://github.com/Ola-Omni/Ola) ``` import os os.environ['LOWRES_RESIZE'] = '384x32' os.environ['HIGHRES_BASE'] = '0x32' os.environ['VIDEO_RESIZE'] = "0x64" os.environ['VIDEO_MAXRES'] = "480" os.environ['VIDEO_MINRES'] = "288" os.environ['MAXRES'] = '1536' os.environ['MINRES'] = '0' os.environ['REGIONAL_POOL'] = '2x' os.environ['FORCE_NO_DOWNSAMPLE'] = '1' os.environ['LOAD_VISION_EARLY'] = '1' os.environ['SKIP_LOAD_VIT'] = '1' import gradio as gr import torch import re from decord import VideoReader, cpu from PIL import Image import numpy as np import transformers import moviepy.editor as mp from typing import Dict, Optional, Sequence, List import librosa import whisper from ola.conversation import conv_templates, SeparatorStyle from ola.model.builder import load_pretrained_model from ola.utils import disable_torch_init from ola.datasets.preprocess import tokenizer_image_token, tokenizer_speech_image_token, tokenizer_speech_question_image_token from ola.mm_utils import get_model_name_from_path, KeywordsStoppingCriteria, process_anyres_video, process_anyres_highres_image_genli from ola.constants import IGNORE_INDEX, DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX, DEFAULT_SPEECH_TOKEN model_path = "" tokenizer, model, image_processor, _ = load_pretrained_model(model_path, None) model = model.to('cuda').eval() model = model.bfloat16() USE_SPEECH=False cur_dir = os.path.dirname(os.path.abspath(__file__)) def load_audio(audio_file_name): speech_wav, samplerate = librosa.load(audio_file_name, sr=16000) if len(speech_wav.shape) > 1: speech_wav = speech_wav[:, 0] speech_wav = speech_wav.astype(np.float32) CHUNK_LIM = 480000 SAMPLE_RATE = 16000 speechs = [] speech_wavs = [] if len(speech_wav) <= CHUNK_LIM: speech = whisper.pad_or_trim(speech_wav) speech_wav = whisper.pad_or_trim(speech_wav) speechs.append(speech) speech_wavs.append(torch.from_numpy(speech_wav).unsqueeze(0)) else: for i in range(0, len(speech_wav), CHUNK_LIM): chunk = speech_wav[i : i + CHUNK_LIM] if len(chunk) < CHUNK_LIM: chunk = whisper.pad_or_trim(chunk) speechs.append(chunk) speech_wavs.append(torch.from_numpy(chunk).unsqueeze(0)) mels = [] for chunk in speechs: chunk = whisper.log_mel_spectrogram(chunk, n_mels=128).permute(1, 0).unsqueeze(0) mels.append(chunk) mels = torch.cat(mels, dim=0) speech_wavs = torch.cat(speech_wavs, dim=0) if mels.shape[0] > 25: mels = mels[:25] speech_wavs = speech_wavs[:25] speech_length = torch.LongTensor([mels.shape[1]] * mels.shape[0]) speech_chunks = torch.LongTensor([mels.shape[0]]) return mels, speech_length, speech_chunks, speech_wavs def extract_audio(videos_file_path): my_clip = mp.VideoFileClip(videos_file_path) return my_clip.audio def ola_inference(multimodal, audio_path): visual, text = multimodal["files"][0], multimodal["text"] if visual.endswith("image2.png"): modality = "video" visual = f"{cur_dir}/case/case1.mp4" if visual.endswith(".mp4"): modality = "video" else: modality = "image" # input audio and video, do not parse audio in the video, else parse audio in the video if audio_path: USE_SPEECH = True elif modality == "video": USE_SPEECH = True else: USE_SPEECH = False speechs = [] speech_lengths = [] speech_wavs = [] speech_chunks = [] if modality == "video": vr = VideoReader(visual, ctx=cpu(0)) total_frame_num = len(vr) fps = round(vr.get_avg_fps()) uniform_sampled_frames = np.linspace(0, total_frame_num - 1, 64, dtype=int) frame_idx = uniform_sampled_frames.tolist() spare_frames = vr.get_batch(frame_idx).asnumpy() video = [Image.fromarray(frame) for frame in spare_frames] else: image = [Image.open(visual)] image_sizes = [image[0].size] if USE_SPEECH and audio_path: audio_path = audio_path speech, speech_length, speech_chunk, speech_wav = load_audio(audio_path) speechs.append(speech.bfloat16().to('cuda')) speech_lengths.append(speech_length.to('cuda')) speech_chunks.append(speech_chunk.to('cuda')) speech_wavs.append(speech_wav.to('cuda')) print('load audio') elif USE_SPEECH and not audio_path: # parse audio in the video audio = extract_audio(visual) audio.write_audiofile("./video_audio.wav") video_audio_path = './video_audio.wav' speech, speech_length, speech_chunk, speech_wav = load_audio(video_audio_path) speechs.append(speech.bfloat16().to('cuda')) speech_lengths.append(speech_length.to('cuda')) speech_chunks.append(speech_chunk.to('cuda')) speech_wavs.append(speech_wav.to('cuda')) else: speechs = [torch.zeros(1, 3000, 128).bfloat16().to('cuda')] speech_lengths = [torch.LongTensor([3000]).to('cuda')] speech_wavs = [torch.zeros([1, 480000]).to('cuda')] speech_chunks = [torch.LongTensor([1]).to('cuda')] conv_mode = "qwen_1_5" if text: qs = text else: qs = '' if USE_SPEECH and audio_path: qs = DEFAULT_IMAGE_TOKEN + "\n" + "User's question in speech: " + DEFAULT_SPEECH_TOKEN + '\n' elif USE_SPEECH: qs = DEFAULT_SPEECH_TOKEN + DEFAULT_IMAGE_TOKEN + "\n" + qs else: qs = DEFAULT_IMAGE_TOKEN + "\n" + qs conv = conv_templates[conv_mode].copy() conv.append_message(conv.roles[0], qs) conv.append_message(conv.roles[1], None) prompt = conv.get_prompt() if USE_SPEECH and audio_path: input_ids = tokenizer_speech_question_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to('cuda') elif USE_SPEECH: input_ids = tokenizer_speech_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to('cuda') else: input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to('cuda') if modality == "video": video_processed = [] for idx, frame in enumerate(video): image_processor.do_resize = False image_processor.do_center_crop = False frame = process_anyres_video(frame, image_processor) if frame_idx is not None and idx in frame_idx: video_processed.append(frame.unsqueeze(0)) elif frame_idx is None: video_processed.append(frame.unsqueeze(0)) if frame_idx is None: frame_idx = np.arange(0, len(video_processed), dtype=int).tolist() video_processed = torch.cat(video_processed, dim=0).bfloat16().to("cuda") video_processed = (video_processed, video_processed) video_data = (video_processed, (384, 384), "video") else: image_processor.do_resize = False image_processor.do_center_crop = False image_tensor, image_highres_tensor = [], [] for visual in image: image_tensor_, image_highres_tensor_ = process_anyres_highres_image_genli(visual, image_processor) image_tensor.append(image_tensor_) image_highres_tensor.append(image_highres_tensor_) if all(x.shape == image_tensor[0].shape for x in image_tensor): image_tensor = torch.stack(image_tensor, dim=0) if all(x.shape == image_highres_tensor[0].shape for x in image_highres_tensor): image_highres_tensor = torch.stack(image_highres_tensor, dim=0) if type(image_tensor) is list: image_tensor = [_image.bfloat16().to("cuda") for _image in image_tensor] else: image_tensor = image_tensor.bfloat16().to("cuda") if type(image_highres_tensor) is list: image_highres_tensor = [_image.bfloat16().to("cuda") for _image in image_highres_tensor] else: image_highres_tensor = image_highres_tensor.bfloat16().to("cuda") pad_token_ids = 151643 attention_masks = input_ids.ne(pad_token_ids).long().to('cuda') stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 keywords = [stop_str] stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) gen_kwargs = {} if "max_new_tokens" not in gen_kwargs: gen_kwargs["max_new_tokens"] = 1024 if "temperature" not in gen_kwargs: gen_kwargs["temperature"] = 0.2 if "top_p" not in gen_kwargs: gen_kwargs["top_p"] = None if "num_beams" not in gen_kwargs: gen_kwargs["num_beams"] = 1 with torch.inference_mode(): if modality == "video": output_ids = model.generate( inputs=input_ids, images=video_data[0][0], images_highres=video_data[0][1], modalities=video_data[2], speech=speechs, speech_lengths=speech_lengths, speech_chunks=speech_chunks, speech_wav=speech_wavs, attention_mask=attention_masks, use_cache=True, stopping_criteria=[stopping_criteria], do_sample=True if gen_kwargs["temperature"] > 0 else False, temperature=gen_kwargs["temperature"], top_p=gen_kwargs["top_p"], num_beams=gen_kwargs["num_beams"], max_new_tokens=gen_kwargs["max_new_tokens"], ) else: output_ids = model.generate( inputs=input_ids, images=image_tensor, images_highres=image_highres_tensor, image_sizes=image_sizes, modalities=['image'], speech=speechs, speech_lengths=speech_lengths, speech_chunks=speech_chunks, speech_wav=speech_wavs, attention_mask=attention_masks, use_cache=True, stopping_criteria=[stopping_criteria], do_sample=True if gen_kwargs["temperature"] > 0 else False, temperature=gen_kwargs["temperature"], top_p=gen_kwargs["top_p"], num_beams=gen_kwargs["num_beams"], max_new_tokens=gen_kwargs["max_new_tokens"], ) outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0] outputs = outputs.strip() if outputs.endswith(stop_str): outputs = outputs[:-len(stop_str)] outputs = outputs.strip() return outputs, None ``` ### Model Architecture - **Architecture:** Pre-trained [Oryx-ViT](https://huggingface.co/THUdyh/Oryx-ViT) + Qwen2.5-7B - **Data:** a mixture of more than 5M image/video/audio data, training for 3 stage. - **Precision:** BFloat16 #### Hardware & Software - **Hardware:** 64 \* NVIDIA Tesla A100 - **Orchestration:** HuggingFace Trainer - **Code:** Pytorch ## Citation @article{liu2025ola, title={Ola: Pushing the Frontiers of Omni-Modal Language Model with Progressive Modality Alignment}, author={Liu, Zuyan and Dong, Yuhao and Wang, Jiahui and Liu, Ziwei and Hu, Winston and Lu, Jiwen and Rao, Yongming}, journal={arXiv preprint arXiv:2502.04328}, year={2025} }