Upload 8 files

app.py

@@ -0,0 +1,144 @@
+import gradio as gr
+from transcription import transcribe_audio
+from translation import translate_text
+
+from process_yt_video import download_audio,get_embed_url
+from create_srt import create_srt
+from custom_theme import CustomTheme
+
+stheme = CustomTheme()
+
+# Function to handle transcription
+def process_transcription(audio_file, youtube_url):
+    if youtube_url:
+        audio_file = download_audio(youtube_url)
+        print(f"Downloaded audio file from YouTube: {audio_file}")
+    
+    if not audio_file:
+        return None, "No audio provided!", None, None
+    
+    print(f"Processing audio file: {audio_file}")
+    detected_lang, transcription = transcribe_audio(audio_file)
+    
+    if not transcription:
+        return "Error in transcription", None
+    
+    return detected_lang, transcription
+
+# Function to handle translation
+def process_translation(transcription, target_lang, detected_lang):
+    if not transcription:
+        return "Please transcribe first!"
+    
+    translated_text = translate_text(transcription, target_lang, detected_lang)
+    return translated_text
+
+# Function to handle subtitle generation
+def process_subtitle(transcription, translation):
+    if not transcription or not translation:
+        return "Please transcribe and translate first!", None
+    
+    subtitle_file = create_srt(transcription, translation)
+    return "Subtitle generated successfully!", subtitle_file
+
+# Function to handle transliteration
+'''def process_transliteration(translated_text):
+    if not translated_text:
+        return "Please translate first!"
+    return "hello"'''
+
+# Function to update embedded YouTube video player
+def update_video(youtube_url):
+    embed_url = get_embed_url(youtube_url)
+    return f"<iframe width='560' height='315' src='{embed_url}' frameborder='0' allowfullscreen></iframe>" if embed_url else ""
+
+with gr.Blocks(theme=stheme) as demo:
+    gr.Markdown("# Voice-to-Text Translation System", elem_id="title")
+    
+    with gr.Row():
+        
+        with gr.Column():
+          gr.Markdown("## Upload Audio or Enter YT URL")
+          audio_input = gr.Audio(sources=["upload","microphone" ], type="filepath", label="Record or Upload Audio 🎤", min_width=50)
+          youtube_url = gr.Textbox(label="Enter YouTube Link", min_width=50)
+          video_player = gr.HTML("")
+          youtube_url.change(update_video, inputs=[youtube_url], outputs=[video_player])
+        
+        with gr.Column():
+          with gr.Row():
+            gr.Markdown("" )
+            transcribe_button = gr.Button("Generate Transcription", interactive=True,size= "sm", min_width=800)
+            detected_language = gr.Textbox(label="Detected Language", interactive=False, min_width=400)
+            transcription_output = gr.Textbox(label="Transcription", interactive=False, min_width=400)
+        
+          with gr.Row():
+            language_selector = gr.Dropdown([
+                'Assamese', 'Bengali', 'Bodo', 'Dogri', 'English', 'Gujarati', 'Hindi', 'Kannada', 'Kashmiri(Perso-Arabic script)', 
+                'Kashmiri(Devanagari script)', 'Konkani', 'Maithili', 'Malayalam', 'Manipuri(Bengali script)', 
+                'Manipuri(Meitei script)', 'Marathi', 'Nepali', 'Odia', 'Punjabi', 'Sanskrit', 'Santali(Ol Chiki script)', 
+                'Sindhi(Perso-Arabic script)', 'Sindhi(Devanagari script)', 'Tamil', 'Telugu', 'Urdu'
+            ], label="Select Target Language", min_width=400)
+          with gr.Row():
+            translate_button = gr.Button("Generate Translation", interactive=True, size= "sm",min_width=350)
+          with gr.Row():
+            translation_output = gr.Textbox(label="Translation", interactive=False, min_width=400)
+        
+          with gr.Row():
+            subtitle_button = gr.Button("Generate Subtitles", interactive=True,size= "sm",min_width=350)
+          with gr.Row():
+            subtitle_status = gr.Textbox(label="Subtitle Status", interactive=False, min_width=400)
+            subtitle_download = gr.File(label="Download Subtitles", visible=True, min_width=400)
+
+
+
+
+    '''with gr.Column():
+        transliterate_button = gr.Button("Generate Transliteration", interactive=True)
+        transliteration_output = gr.Textbox(label="Transliteration", interactive=False)'''
+    
+    transcribe_button.click(
+        process_transcription, 
+        inputs=[audio_input, youtube_url],
+        outputs=[detected_language, transcription_output]
+    )
+    
+    translate_button.click(
+        process_translation, 
+        inputs=[transcription_output, language_selector, detected_language], 
+        outputs=[translation_output]
+    )
+    
+    subtitle_button.click(
+        process_subtitle, 
+        inputs=[transcription_output, translation_output], 
+        outputs=[subtitle_status, subtitle_download]
+    )
+    
+    '''transliterate_button.click(
+        process_transliteration, 
+        inputs=[translation_output], 
+        outputs=[transliteration_output]
+    )'''
+# Add CSS for custom styling
+demo.css = """
+#title {
+    text-align: center;
+    font-size: 36px;
+    font-weight: bold;
+    width: 100%;
+}
+
+#label-center {
+    text-align: center;
+    font-size: 18px;
+    font-weight: bold;
+    width: 50%;
+    margin: auto;
+
+}
+
+
+"""
+# Launch the Gradio Apps
+if __name__ == "__main__":
+    demo.launch(share=True, debug=True,pwa=True)

create_srt.py

@@ -0,0 +1,16 @@
+  
+def create_srt(transcription, translated_text):
+    srt_content = ""
+    for idx, (trans, trans_tr) in enumerate(zip(transcription.split("\n"), translated_text.split("\n"))):
+        start_time = f"00:00:{idx:02d},000"
+        end_time = f"00:00:{(idx + 1):02d},000"
+        srt_content += f"{idx+1}\n{start_time} --> {end_time}\n{trans}\n{trans_tr}\n\n"
+
+    # Save to a file
+    subtitle_file = "translated_subtitles.srt"
+    with open(subtitle_file, "w", encoding="utf-8") as f:
+        f.write(srt_content)
+
+    return subtitle_file
+    
+    

custom_theme.py

@@ -0,0 +1,69 @@
+
+from __future__ import annotations
+from typing import Iterable
+
+from gradio.themes.base import Base
+from gradio.themes.utils import colors, fonts, sizes
+
+
+class CustomTheme(Base):
+    def __init__(
+        self,
+        *,
+        primary_hue: colors.Color | str = colors.rose,
+        secondary_hue: colors.Color | str = colors.amber,
+        neutral_hue: colors.Color | str = colors.gray,
+        spacing_size: sizes.Size | str = sizes.spacing_md,
+        radius_size: sizes.Size | str = sizes.radius_md,
+        text_size: sizes.Size | str = sizes.text_lg,
+        font: fonts.Font
+        | str
+        | Iterable[fonts.Font | str] = (
+            fonts.GoogleFont("Quicksand"),
+            "ui-sans-serif",
+            "sans-serif",
+        ),
+        font_mono: fonts.Font
+        | str
+        | Iterable[fonts.Font | str] = (
+            fonts.GoogleFont("IBM Plex Mono"),
+            "ui-monospace",
+            "monospace",
+        ),
+    ):
+        super().__init__(
+            primary_hue=primary_hue,
+            secondary_hue=secondary_hue,
+            neutral_hue=neutral_hue,
+            spacing_size=spacing_size,
+            radius_size=radius_size,
+            text_size=text_size,
+            font=font,
+            font_mono=font_mono,
+        )
+        super().set(
+            # 🌅 **New Elegant Background**
+            body_background_fill="""
+                radial-gradient(circle at top left, *primary_200, *secondary_100),
+                linear-gradient(120deg, *primary_300, *secondary_200)
+            """,
+            body_background_fill_dark="""
+                radial-gradient(circle at bottom right, *primary_800, *secondary_600),
+                linear-gradient(120deg, *primary_900, *secondary_700)
+            """,
+
+            # 🔘 Dark Grey Buttons with Hover Effect
+            button_primary_background_fill="#4A4A4A",
+            button_primary_background_fill_hover="#6A6A6A",
+            button_primary_text_color="white",
+            button_primary_background_fill_dark="#3A3A3A",
+            button_primary_shadow="0px 4px 12px rgba(0,0,0,0.3)",
+
+            # 🖱️ Other UI Elements
+            slider_color="*secondary_300",
+            slider_color_dark="*secondary_600",
+            block_title_text_weight="600",
+            block_border_width="3px",
+            block_shadow="*shadow_drop_lg",
+            button_large_padding="32px",
+        )

install.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+echo "Updating package list..."
+sudo apt update
+
+echo "Installing required system dependencies..."
+sudo apt install -y git wget ffmpeg  # Added ffmpeg
+
+
+echo "Upgrading pip..."
+pip install --upgrade pip
+
+echo "Installing Python dependencies..."
+pip install --upgrade --no-deps --force-reinstall git+https://github.com/openai/whisper.git
+pip install transformers torch numpy gitpython  
+pip install whisper  # Explicitly installing Whisper
+pip install bitsandbytes accelerate 
+pip install gradio
+pip install yt_dlp tiktoken
+echo "Cloning IndicTrans2 repository..."
+if [ ! -d "IndicTrans2" ]; then
+    git clone https://github.com/AI4Bharat/IndicTrans2
+else
+    echo "IndicTrans2 already exists, skipping clone."
+fi
+
+echo "Navigating to IndicTrans2 directory..."
+cd IndicTrans2/huggingface_interface || exit
+
+echo "Running IndicTrans2 install script..."
+bash install.sh
+cd..
+echo "Setup complete! You can now use Whisper, IndicTrans2, and Gradio."

process_yt_video.py

@@ -0,0 +1,35 @@
+# Function to extract video ID from YouTube URL
+import re
+import yt_dlp
+
+def get_video_id(youtube_url):
+    match = re.search(r"(?:v=|\/)([a-zA-Z0-9_-]{11})", youtube_url)    
+    return match.group(0) if match else None
+
+# Function to generate YouTube embed URL
+def get_embed_url(youtube_url):
+    video_id = get_video_id(youtube_url)
+    if video_id:
+        return f"https://www.youtube.com/embed/{video_id}"
+    return None
+# Function to download audio 
+def download_audio(youtube_url):
+    video_id = get_video_id(youtube_url)
+    if not video_id:
+
+        return None, None  # Invalid URL
+
+    
+
+    ydl_opts = {
+        'format': 'bestaudio/best',
+        'outtmpl': 'temp_audio.%(ext)s',
+        'postprocessors': [{'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192'}],
+        'quiet': True,
+    }
+
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        info = ydl.extract_info(youtube_url, download=True)
+        audio_path = "temp_audio.mp3"  
+
+    return audio_path

transcription.py

@@ -0,0 +1,52 @@
+import whisper
+import numpy as np
+
+from whisper import load_model, transcribe
+from whisper.audio import load_audio
+
+def transcribe_audio(audio):
+  model = load_model("small")
+  #audio_path = "/content/bharat.mp3"
+  #audio = load_audio(audio_path)
+  result = transcribe(model, audio)
+
+
+  detected_language = result.get("language")
+  whisper_to_indictrans2 = {
+    "as": "asm_Beng",   # Assamese
+    "bn": "ben_Beng",   # Bengali
+    "brx": "brx_Deva",  # Bodo
+    "doi": "doi_Deva",  # Dogri
+    "gu": "guj_Gujr",   # Gujarati
+    "hi": "hin_Deva",   # Hindi
+    "kn": "kan_Knda",   # Kannada
+    "ks": "kas_Arab",   # Kashmiri (Perso-Arabic script)
+    "ks_Deva": "kas_Deva",  # Kashmiri (Devanagari script)
+    "kok": "kok_Deva",  # Konkani
+    "mai": "mai_Deva",  # Maithili
+    "ml": "mal_Mlym",   # Malayalam
+    "mni": "mni_Beng",  # Manipuri (Bengali script)
+    "mni_Mtei": "mni_Mtei",  # Manipuri (Meitei script)
+    "mr": "mar_Deva",   # Marathi
+    "ne": "nep_Deva",   # Nepali
+    "or": "ory_Orya",   # Odia
+    "pa": "pan_Guru",   # Punjabi
+    "sa": "san_Deva",   # Sanskrit
+    "sat": "sat_Olck",  # Santali (Ol Chiki script)
+    "sd": "snd_Arab",   # Sindhi (Perso-Arabic script)
+    "sd_Deva": "snd_Deva",  # Sindhi (Devanagari script)
+    "ta": "tam_Taml",   # Tamil
+    "te": "tel_Telu",   # Telugu
+    "ur": "urd_Arab",    # Urdu
+    "en": "eng_Latn",
+}
+  if detected_language in whisper_to_indictrans2.keys():
+    detected_language = whisper_to_indictrans2[detected_language]
+  elif detected_language not in whisper_to_indictrans2.keys():
+    return "Unknown language detected",None
+    
+  
+  transcription = result.get("text")  # Adjust key if necessary
+
+  return detected_language, transcription
+

translation.py

@@ -0,0 +1,219 @@
+import torch
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+import sys
+import os
+
+# Get the absolute path of IndicTransToolkit
+indictrans_path = "/content/Voice-to-Text-Translation-System-Leveraging-Whisper-and-IndicTrans2/IndicTrans2/huggingface_interface/IndicTransToolkit/IndicTransToolkit"
+sys.path.append(indictrans_path)
+
+from processor import IndicProcessor
+
+# Check if GPU is available
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+def translate_text(transcription, target_lang, src_lang):
+    mapping = {
+        "Assamese": "asm_Beng", "Bengali": "ben_Beng", "Bodo": "brx_Deva", "Dogri": "doi_Deva",
+        "Gujarati": "guj_Gujr", "Hindi": "hin_Deva", "Kannada": "kan_Knda",
+        "Kashmiri(Perso-Arabic script)": "kas_Arab", "Kashmiri(Devanagari script)": "kas_Deva",
+        "Konkani": "kok_Deva", "Maithili": "mai_Deva", "Malayalam": "mal_Mlym",
+        "Manipuri(Bengali script)": "mni_Beng", "Manipuri(Meitei script)": "mni_Mtei",
+        "Marathi": "mar_Deva", "Nepali": "nep_Deva", "Odia": "ory_Orya",
+        "Punjabi": "pan_Guru", "Sanskrit": "san_Deva", "Santali(Ol Chiki script)": "sat_Olck",
+        "Sindhi(Perso-Arabic script)": "snd_Arab", "Sindhi(Devanagari script)": "snd_Deva",
+        "Tamil": "tam_Taml", "Telugu": "tel_Telu", "Urdu": "urd_Arab","English":"eng_Latn",
+    }
+    if target_lang in mapping:
+      tgt_lang = mapping[target_lang]
+
+    if src_lang == tgt_lang:
+      return "Detected Language and Target Language cannot be same"
+
+    if src_lang == "eng_Latn":
+      model_name = "prajdabre/rotary-indictrans2-en-indic-1B"
+    else:
+      model1_name ="prajdabre/rotary-indictrans2-indic-en-1B"
+      model2_name = "prajdabre/rotary-indictrans2-en-indic-1B"
+      translations = indic_indic(model1_name,model2_name, src_lang, target_lang,transcription)
+      return translations 
+    
+    
+    
+    
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+    # Load model in 8-bit quantization
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        model_name,
+        trust_remote_code=True,
+        torch_dtype=torch.float16,
+        #load_in_8bit=True, 
+        attn_implementation="flash_attention_2"
+    ).to(DEVICE)
+
+    ip = IndicProcessor(inference=True)
+
+    input_sentences = [transcription]
+
+    batch = ip.preprocess_batch(input_sentences, src_lang=src_lang, tgt_lang=tgt_lang)
+
+    
+    # Tokenize the sentences and generate input encodings
+    inputs = tokenizer(
+        batch,
+        truncation=True,
+        padding="longest",
+        return_tensors="pt",
+        max_length=2048,
+        
+    )
+
+    # Move inputs to the correct device (only inputs, NOT model)
+    inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
+
+    # Generate translations using the model
+    with torch.inference_mode():
+        generated_tokens = model.generate(
+            **inputs,
+            num_beams=5,
+        length_penalty=1.5,
+        repetition_penalty=2.0,
+        num_return_sequences=1,
+        max_new_tokens=2048,
+        early_stopping=True
+        )
+
+    # Move generated tokens to CPU before decoding
+    generated_tokens = generated_tokens.cpu().tolist()
+   
+    # Decode the generated tokens into text
+    with tokenizer.as_target_tokenizer():
+        generated_tokens = tokenizer.batch_decode(
+            generated_tokens,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=True
+        )
+
+    # Postprocess the translations
+    translations = ip.postprocess_batch(generated_tokens, lang=tgt_lang)
+    print(type(translations))
+    translations =str(translations).strip("'")
+    return translations
+def indic_indic(model1_name,model2_name,src_lang,tgt_lang,transcription,intermediate_lng ="eng_Latn",):
+  tokenizer = AutoTokenizer.from_pretrained(model1_name, trust_remote_code=True)
+
+    # Load model in 8-bit quantization
+  model = AutoModelForSeq2SeqLM.from_pretrained(
+        model1_name,
+        trust_remote_code=True,
+        torch_dtype=torch.float16,
+        #load_in_8bit=True, 
+        attn_implementation="flash_attention_2"
+    ).to(DEVICE)
+
+  ip = IndicProcessor(inference=True)
+
+  input_sentences = [transcription]
+
+  batch = ip.preprocess_batch(input_sentences, src_lang=src_lang, tgt_lang=intermediate_lng)
+
+    # Tokenize the sentences and generate input encodings
+  inputs = tokenizer(
+        batch,
+        truncation=True,
+        padding="longest",
+        return_tensors="pt",
+        max_length=2048,
+    )
+
+    # Move inputs to the correct device (only inputs, NOT model)
+  inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
+
+    # Generate translations using the model
+  with torch.inference_mode():
+        generated_tokens = model.generate(
+            **inputs,
+            num_beams=10,
+        length_penalty=1.5,
+        repetition_penalty=2.0,
+        num_return_sequences=1,
+        max_new_tokens=2048,
+        early_stopping=True
+        )
+
+    # Move generated tokens to CPU before decoding
+  generated_tokens = generated_tokens.cpu().tolist()
+
+    # Decode the generated tokens into text
+  with tokenizer.as_target_tokenizer():
+        generated_tokens = tokenizer.batch_decode(
+            generated_tokens,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=True
+        )
+
+    # Postprocess the translations
+  translations1 = ip.postprocess_batch(generated_tokens, lang=tgt_lang)
+  
+  translations1 =str(translations).strip("'")
+  tokenizer = AutoTokenizer.from_pretrained(model2_name, trust_remote_code=True)
+
+    # Load model in 8-bit quantization
+  model = AutoModelForSeq2SeqLM.from_pretrained(
+        model2_name,
+        trust_remote_code=True,
+        torch_dtype=torch.float16,
+        #load_in_8bit=True, 
+        attn_implementation="flash_attention_2"
+    ).to(DEVICE)
+
+  ip = IndicProcessor(inference=True)
+
+  input_sentences = [translations1]
+
+  batch = ip.preprocess_batch(input_sentences, src_lang=intermediate_lng, tgt_lang=tgt_lang)
+
+    # Tokenize the sentences and generate input encodings
+  inputs = tokenizer(
+        batch,
+        truncation=True,
+        padding="longest",
+        return_tensors="pt",
+        max_length=2048,
+    )
+
+    # Move inputs to the correct device (only inputs, NOT model)
+  inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
+
+    # Generate translations using the model
+  with torch.inference_mode():
+        generated_tokens = model.generate(
+            **inputs,
+            num_beams=10,
+        length_penalty=1.5,
+        repetition_penalty=2.0,
+        num_return_sequences=1,
+        max_new_tokens=2048,
+        early_stopping=True
+        )
+ 
+
+    # Move generated tokens to CPU before decoding
+  generated_tokens = generated_tokens.cpu().tolist()
+
+    # Decode the generated tokens into text
+  with tokenizer.as_target_tokenizer():
+        generated_tokens = tokenizer.batch_decode(
+            generated_tokens,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=True
+        )
+
+    # Postprocess the translations
+  translations = ip.postprocess_batch(generated_tokens, lang=tgt_lang)
+  
+
+  return translations
+

transliteration.py

@@ -0,0 +1 @@
+#Under development