File size: 2,412 Bytes
83eb24a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import gradio as gr
import torch
import torch.nn.functional as F
from transformers import AutoTokenizer
from unsloth import FastLanguageModel
# Model Setup
max_seq_length = 2048
load_in_4bit = True
name = "large-traversaal/Phi-4-Hindi"
model, tokenizer = FastLanguageModel.from_pretrained(
model_name=name,
max_seq_length=max_seq_length,
load_in_4bit=load_in_4bit,
)
model = FastLanguageModel.get_peft_model(
model,
r=16,
target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
FastLanguageModel.for_inference(model)
def generate_response(task, input_text, temperature, top_p, max_tokens):
prompt = f"### INPUT : {input_text} RESPONSE : "
message = [{"role": "user", "content": prompt}]
inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors="pt").to("cuda")
outputs = model.generate(
input_ids=inputs,
max_new_tokens=max_tokens,
use_cache=True,
temperature=temperature,
top_p=top_p,
pad_token_id=tokenizer.eos_token_id,
)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
processed_response = response.split("### RESPONSE :assistant")[-1].strip()
return processed_response
# Gradio Interface
def gradio_ui():
with gr.Blocks() as demo:
gr.Markdown("## Test Space: Chat with Phi-4-Hindi")
with gr.Row():
task = gr.Dropdown([
"Long Response", "Short Response", "NLI", "Translation", "MCQ", "Cross-Lingual"
], label="Select Task")
input_text = gr.Textbox(label="Input Text")
with gr.Row():
temperature = gr.Slider(0.1, 1.0, value=0.7, step=0.1, label="Temperature")
top_p = gr.Slider(0.1, 1.0, value=0.9, step=0.1, label="Top P")
max_tokens = gr.Slider(50, 800, value=200, step=50, label="Max Tokens")
output_text = gr.Textbox(label="Generated Response")
btn = gr.Button("Generate")
btn.click(generate_response, inputs=[task, input_text, temperature, top_p, max_tokens], outputs=output_text)
return demo
# Launch Gradio App
demo = gradio_ui()
demo.launch() |