app.py

import gradio as gr
from huggingface_hub import InferenceClient
import os


system_data = os.getenv("SYSTEM_DATA")

client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")

def respond(
    message,
    history: list[tuple[str, str]],
    system_message,
    max_tokens,
    temperature,
    top_p,
):
    system_message = str(system_message) if system_message else ""
    system_message += f"\n{system_data}"

    messages = [{"role": "system", "content": system_message}]
    
    for val in history:
        if val[0]:
            messages.append({"role": "user", "content": str(val[0])})
        if val[1]:
            messages.append({"role": "assistant", "content": str(val[1])})
    
    messages.append({"role": "user", "content": str(message)})

    response = ""
    try:
        for msg in client.chat_completion(
            messages,
            max_tokens=max_tokens,
            stream=True,
            temperature=temperature,
            top_p=top_p,
        ):
            token = msg.choices[0].delta.content
            response += token
            yield response
    except Exception as e:
        yield f"Error: {e}"

demo = gr.ChatInterface(respond, theme="Yntec/HaleyCH_Theme_Orange")

if __name__ == "__main__":
    demo.launch()