Update app.py

app.py

@@ -1,10 +1,22 @@
+import torch
+import os
 import gradio as gr
-from huggingface_hub import InferenceClient
+from huggingface_hub import InferenceClient, client
+# Use a pipeline as a high-level helper
+from transformers import BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16)
+model_4bit = AutoModelForCausalLM.from_pretrained(
+    "unsloth/Llama-4-Scout-17B-16E-Instruct-unsloth-bnb-4bit",
+    quantization_config=quantization_config,
+    torch_dtype="auto"
+)
+# pipe = pipeline("image-text-to-text", model="")
+# pipe(messages)
+
+client = client(model_4bit)
 
 
 def respond(
@@ -15,6 +27,9 @@ def respond(
     temperature,
     top_p,
 ):
+    messages = [
+    {"role": "user", "content": "Who are you?"},
+]
     messages = [{"role": "system", "content": system_message}]
 
     for val in history: