Add automatic speech recognition (ASR) sample

app.py

@@ -2,6 +2,7 @@ from dotenv import load_dotenv
 from functools import partial
 import gradio as gr
 from huggingface_hub import InferenceClient
+from automatic_speech_recognition import automatic_speech_recognition
 from image_classification import image_classification
 from image_to_text import image_to_text
 from text_to_image import text_to_image
@@ -64,7 +65,7 @@ class App:
                         outputs=image_classification_output
                     )
                 with gr.Tab("Text-to-speech (TTS)"):
-                    gr.Markdown("Generate speech from a text.")
+                    gr.Markdown("Generate speech from text.")
                     text_to_speech_text = gr.Textbox(label="Text")
                     text_to_speech_generate_button = gr.Button("Generate")
                     text_to_speech_output = gr.Audio(label="Speech")
@@ -73,6 +74,16 @@ class App:
                         inputs=text_to_speech_text,
                         outputs=text_to_speech_output
                     )
+                with gr.Tab("Audio Transcription or Automatic Speech Recognition (ASR)"):
+                    gr.Markdown("Transcribe audio to text.")
+                    audio_transcription_audio_input = gr.Audio(label="Audio")
+                    audio_transcription_generate_button = gr.Button("Transcribe")
+                    audio_transcription_output = gr.Textbox(label="Text")
+                    audio_transcription_generate_button.click(
+                        fn=automatic_speech_recognition,
+                        inputs=audio_transcription_audio_input,
+                        outputs=audio_transcription_output
+                    )
 
             demo.launch()
 

automatic_speech_recognition.py

@@ -0,0 +1,14 @@
+import gc
+from os import getenv
+from transformers import pipeline
+from utils import spaces_gpu, resample_audio
+
+
+@spaces_gpu
+def automatic_speech_recognition(audio: tuple[int, bytes]) -> str:
+    asr = pipeline(task="automatic-speech-recognition", model=getenv("AUDIO_TRANSCRIPTION_MODEL"))
+    audio_array = resample_audio(asr.feature_extractor.sampling_rate, audio)
+    result = asr(audio_array)
+    del asr
+    gc.collect()
+    return result["text"]

requirements.txt

@@ -1,10 +1,13 @@
 gradio>=5.49.1
 huggingface-hub>=0.34.0,<1.0
-python-dotenv>=1.0.0
+inflect>=7.0.0
+librosa>=0.10.0
+numpy>=1.24.0
 pandas>=2.0.0
+phonemizer>=3.0.0
 pillow>=10.0.0
+python-dotenv>=1.0.0
 requests>=2.31.0
-transformers>=4.40.0
+soundfile>=0.12.0
 timm>=1.0.0
-inflect>=7.0.0
-phonemizer>=3.0.0
+transformers>=4.40.0

text_to_speech.py

@@ -1,10 +1,12 @@
 import gc
+from os import getenv
 from transformers import pipeline
 from utils import spaces_gpu
 
+
 @spaces_gpu
 def text_to_speech(text: str) -> tuple[int, bytes]:
-    narrator = pipeline("text-to-speech", "kakao-enterprise/vits-ljs")
+    narrator = pipeline("text-to-speech", getenv("TEXT_TO_SPEECH_MODEL"))
     del narrator
     gc.collect()
     result = narrator(text)

utils.py

@@ -5,12 +5,14 @@ from os import getenv
 import requests
 from tempfile import NamedTemporaryFile
 import torch
+import numpy as np
+import soundfile as sf
+import librosa
 
 
 # Try to import spaces decorator (for Hugging Face Spaces), otherwise use no-op decorator.
 try:
-    import spaces
-    spaces_gpu = spaces.GPU
+    from spaces import GPU as spaces_gpu
 except ImportError:
     # For local development, use a no-op decorator because spaces is not available.
     def spaces_gpu(func):
@@ -42,3 +44,20 @@ def save_image_to_temp_file(image: Image) -> str:
     temp_file.close()
     image.save(temp_path, format=image_format)
     return temp_path
+
+def resample_audio(target_sample_rate: int, audio: tuple[int, bytes | np.ndarray]) -> np.ndarray:
+    sample_rate, audio_data = audio
+    
+    # Convert audio data to a numpy array if it’s bytes
+    if isinstance(audio_data, bytes):
+        audio_array = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) / 32768.0
+    elif isinstance(audio_data, np.ndarray):
+        audio_array = audio_data.astype(np.float32)
+    else:
+        raise ValueError(f"Unsupported audio_data type: {type(audio_data)}")
+    
+    # Resample if sample rates don’t match.
+    if sample_rate != target_sample_rate:
+        audio_array = librosa.resample(audio_array, orig_sr=sample_rate, target_sr=target_sample_rate)
+    
+    return audio_array