Create only_long_response.py

only_long_response.py

@@ -0,0 +1,66 @@
+import gradio as gr
+import torch
+from unsloth import FastLanguageModel
+from transformers import AutoTokenizer
+
+def load_model():
+    model_name = "large-traversaal/Phi-4-Hindi"
+    max_seq_length = 2048
+    load_in_4bit = True
+    
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=model_name, 
+        max_seq_length=max_seq_length, 
+        load_in_4bit=load_in_4bit,
+    )
+    
+    model = FastLanguageModel.get_peft_model(
+        model,
+        r=16, 
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+        lora_alpha=16, 
+        lora_dropout=0, 
+        bias="none", 
+        use_gradient_checkpointing="unsloth",
+        random_state=3407, 
+        use_rslora=False,
+        loftq_config=None,
+    )
+    FastLanguageModel.for_inference(model)
+    return model, tokenizer
+
+# Load model and tokenizer
+model, tokenizer = load_model()
+
+def generate_response(input_text):
+    prompt = f"### INPUT : {input_text} RESPONSE : "
+    message = [{"role": "user", "content": prompt}]
+    
+    inputs = tokenizer.apply_chat_template(
+        message, tokenize=True, add_generation_prompt=True, return_tensors="pt"
+    ).to("cuda")
+    
+    outputs = model.generate(
+        input_ids=inputs, 
+        max_new_tokens=1000, 
+        use_cache=True, 
+        temperature=0.1, 
+        min_p=0.1, 
+        pad_token_id=tokenizer.eos_token_id
+    )
+    
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    processed_response = response.split("### RESPONSE :assistant")[-1].strip()
+    
+    return processed_response
+
+# Gradio UI
+iface = gr.Interface(
+    fn=generate_response,
+    inputs=gr.Textbox(lines=5, placeholder="Enter your query here..."),
+    outputs="text",
+    title="AI Hindi Chatbot",
+    description="Enter your prompt and get a response generated by the AI model."
+)
+
+iface.launch()