Implement streaming inference display

- Added TextIteratorStreamer for real-time token streaming
- Response appears word-by-word as it's generated
- Added typing indicator (●) during generation
- Improved user experience with immediate visual feedback
- Used threading for non-blocking generation
- Added CSS animation for typing indicator
- Shows '🔄 Starting...' initially, then streams response
- Button shows '⏳ Generating...' during streaming

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
 import spaces
 
 # Model configuration
@@ -67,7 +68,7 @@ def format_prompt(message, history):
     return prompt
 
 @spaces.GPU(duration=60)
-def generate_response(
+def generate_response_streaming(
     message,
     history,
     temperature=0.3,
@@ -75,7 +76,7 @@ def generate_response(
     top_p=0.95,
     repetition_penalty=1.05,
 ):
-    """Generate response from the model"""
+    """Generate response from the model with streaming"""
     
     # Format the prompt
     prompt = format_prompt(message, history)
@@ -89,23 +90,38 @@ def generate_response(
         if k != 'token_type_ids':  # Filter out token_type_ids
             model_inputs[k] = v.to(model.device)
     
-    # Generate response
-    with torch.no_grad():
-        outputs = model.generate(
-            **model_inputs,
-            max_new_tokens=max_new_tokens,
-            temperature=temperature,
-            top_p=top_p,
-            repetition_penalty=repetition_penalty,
-            do_sample=True,
-            pad_token_id=tokenizer.pad_token_id,
-            eos_token_id=tokenizer.eos_token_id,
-        )
+    # Set up the streamer
+    streamer = TextIteratorStreamer(
+        tokenizer,
+        skip_prompt=True,
+        skip_special_tokens=True,
+        timeout=30.0
+    )
+    
+    # Generation parameters
+    generation_kwargs = dict(
+        **model_inputs,
+        max_new_tokens=max_new_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        streamer=streamer,
+    )
+    
+    # Start generation in a separate thread
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
     
-    # Decode response
-    response = tokenizer.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
+    # Stream the response
+    partial_response = ""
+    for new_text in streamer:
+        partial_response += new_text
+        yield partial_response
     
-    return response
+    thread.join()
 
 # Question categories for the carousel
 QUESTION_CATEGORIES = {
@@ -415,6 +431,17 @@ custom_css = """
     animation: pulse 1.5s ease-in-out infinite;
 }
 
+/* Typing indicator */
+@keyframes typing {
+    0%, 60%, 100% { opacity: 0.3; }
+    30% { opacity: 1; }
+}
+
+.typing-indicator {
+    display: inline-block;
+    animation: typing 1.4s infinite;
+}
+
 .question-button:last-child {
     margin-bottom: 0;
 }
@@ -852,20 +879,26 @@ with gr.Blocks(theme=gr.themes.Soft(), css=custom_css) as demo:
             return "", chat_history, gr.update(value="Send Question")
         
         try:
-            # Show processing state
-            yield "", chat_history + [(message, "🔄 Processing...")], gr.update(value="⏳ Generating...")
+            # Show initial processing state
+            yield "", chat_history + [(message, "🔄 Starting...")], gr.update(value="⏳ Generating...")
             
-            response = generate_response(
+            # Stream the response
+            partial_response = ""
+            for chunk in generate_response_streaming(
                 message, 
                 chat_history,
                 temperature,
                 max_new_tokens,
                 top_p,
                 repetition_penalty
-            )
+            ):
+                partial_response = chunk
+                # Update chat with partial response and typing indicator
+                current_history = chat_history + [(message, partial_response + " ●")]
+                yield "", current_history, gr.update(value="⏳ Generating...")
             
-            # Update with actual response
-            chat_history.append((message, response))
+            # Final update with complete response
+            chat_history.append((message, partial_response))
             yield "", chat_history, gr.update(value="Send Question")
             
         except Exception as e: