vectostoreindex

.gitignore

@@ -2,3 +2,4 @@
 __pycache__/
 appcompleta.py
 appLlama.py
+interface.py

app.py

@@ -9,6 +9,7 @@ from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorS
 from huggingface_hub import login
 from llama_index.core import Settings, VectorStoreIndex, SimpleDirectoryReader, ChatPromptTemplate, PromptTemplate, load_index_from_storage, StorageContext
 from llama_index.core.node_parser import SentenceSplitter
+from llama_index.embeddings.instructor import InstructorEmbedding
 
 
 huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
@@ -41,6 +42,9 @@ model.config.sliding_window = 4096
 #model = model.to(device)
 model.eval()
 
+Settings.embed_model = InstructorEmbedding(model_name="hkunlp/instructor-base")
+
+
 ###------####
 # rag
 documents_paths = {

appLlama.py

@@ -1,29 +0,0 @@
-from backend import handle_query
-import gradio as gr
-
-
-DESCRIPTION = """\
-# <div style="text-align: center;">Odi, l'assistente ricercatore degli Osservatori</div>
-
-
-👉 Retrieval-Augmented Generation - Ask me anything about the research carried out at the Osservatori.
-"""
-
-
-chat_interface =gr.ChatInterface(
-        fn=handle_query, 
-        chatbot=gr.Chatbot(height=500),
-        textbox=gr.Textbox(placeholder="Chiedimi qualasiasi cosa relativa agli Osservatori", container=False, scale=7),
-        #examples=[["Ciao, in cosa puoi aiutarmi?"],["Dimmi i risultati e le modalità di conduzione del censimento per favore"]]
-        )
-
-    
-with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}") as demo:
-    gr.Markdown(DESCRIPTION)
-    #gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
-    chat_interface.render() 
-    
-if __name__ == "__main__":
-    #progress = gr.Progress(track_tqdm=True)
-    demo.launch()
-       

appcompleta.py

@@ -1,262 +0,0 @@
-import os
-import spaces
-from threading import Thread
-from typing import Iterator
-from backend2 import load_documents, prepare_documents, get_context_sources
-import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
-from huggingface_hub import login
-import threading
-
-
-huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
-login(huggingface_token)
-
-DESCRIPTION = """\
-# La Chatbot degli Osservatori
-"""
-MAX_MAX_NEW_TOKENS = 2048
-DEFAULT_MAX_NEW_TOKENS = 1024
-os.environ["MAX_INPUT_TOKEN_LENGTH"] = "4096" #"8192"
-MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH"))
-
-
-# Force usage of CPU
-#device = torch.device("cpu")
-
-device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-
-model_id = "google/gemma-2-2b-it"
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    device_map="auto",
-    torch_dtype= torch.float16 if torch.cuda.is_available() else torch.float32,
-)
-tokenizer = GemmaTokenizerFast.from_pretrained(model_id)
-#tokenizer = AutoTokenizer.from_pretrained(model_id)
-tokenizer.use_default_system_prompt = False
-model.config.sliding_window = 4096
-#model = model.to(device)
-model.eval()
-
-###------####
-# rag
-documents_paths = {
-    'blockchain': 'documents/blockchain',
-    'metaverse': 'documents/metaverso',
-    'payment': 'documents/payment'
-}
-
-"""session_state = {"documents_loaded": False, 
-                 "document_db": None, 
-                 "original_message": None, 
-                 "clarification": False}"""
-
-INSTRUCTION_1 = 'In italiano, chiedi molto brevemente se la domanda si riferisce agli "Osservatori Blockchain", "Osservatori Payment" oppure "Osservatori Metaverse".'
-INSTRUCTION_2 = 'Sei un assistente che risponde in italiano alle domande basandosi solo sulle informazioni fornite nel contesto. Se non trovi informazioni, rispondi "Puoi chiedere maggiori informazioni all\'ufficio di riferimento.". Se invece la domanda è completamente fuori contesto, non rispondere e rammenta il topic del contesto'
-
-default_error_response = (
-    'Non sono sicuro che tu voglia indirizzare la tua ricerca su una di queste opzioni: '
-    '"Blockchain", "Metaverse", "Payment". '
-    'Per favore utilizza il nome corretto.'
-)
-
-
-thread_local = threading.local()
-
-def get_session_state():
-    if not hasattr(thread_local, "session_state"):
-        thread_local.session_state = {
-            "documents_loaded": False,
-            "document_db": None,
-            "original_message": None,
-            "clarification": False
-        }
-    return thread_local.session_state
-
-@spaces.GPU(duration=30)
-def generate(
-    message: str,
-    chat_history: list[tuple[str, str]],
-    max_new_tokens: int = 1024,
-    temperature: float = 0.6,
-    top_p: float = 0.9,
-    top_k: int = 50,
-    repetition_penalty: float = 1.2,
-) -> Iterator[str]:
-    
-    session_state = get_session_state()
-    
-    global context, sources, conversation
-    
-    if (not (session_state["documents_loaded"]) and not (session_state["clarification"])):
-        
-        conversation = []
-        for user, assistant in chat_history:
-            conversation.extend(
-                [
-                    {"role": "user", "content": user},
-                    {"role": "assistant", "content": assistant},
-                ]
-            )
-        conversation.append({"role": "user", "content": f"Domanda: {message} . Comando: {INSTRUCTION_1}" })
-        conversation.append({"role": "assistant", "content": "Ok."})   
-        print("debug - CONV1", conversation)
-        
-        session_state["original_message"] = message
-        session_state["clarification"] = True       
-
-        
-    elif session_state["clarification"]:
-
-        message = message.lower()
-        matched_path = None
-        
-        for key, path in documents_paths.items():
-             if key in message:
-                matched_path = path
-                break
-            
-        if matched_path:
-            yield "Fammi cercare tra i miei documenti..." 
-            documents = load_documents(matched_path)
-            session_state["document_db"] = prepare_documents(documents)
-            session_state["documents_loaded"] = True
-            yield f"Ecco, ho raccolto informazioni dagli Osservatori {key.capitalize()}. Ora sto elaborando una risposta per te..."
-            sources = [] 
-            context, sources = get_context_sources(session_state["original_message"], session_state["document_db"])
-            print("sources ", sources)
-            print("context ", context)
-            
-            conversation = []
-            conversation.append({"role": "user", "content": f"{INSTRUCTION_2}"})
-            conversation.append({"role": "assistant", "content": "Ok."}) 
-            for user, assistant in chat_history:
-                conversation.extend(
-                    [
-                        {"role": "user", "content": user },
-                        {"role": "assistant", "content": assistant},
-                    ]
-                )      
-            conversation.append({"role": "user", "content": f'Contesto: {context}\n\n Domanda iniziale: {session_state["original_message"]} . Rispondi solo in italiano.'})
-            session_state["clarification"] = False 
-            print("debug - CONV2", conversation)
-         
-        else:
-             print(default_error_response)
-             gr.Info("NO MATCH")
-        
-    else:
-        conversation = []
-        conversation.append({"role": "user", "content": f"Comandi: {INSTRUCTION_2}"})
-        conversation.append({"role": "assistant", "content": "Va bene."})
-        for user, assistant in chat_history:
-            conversation.extend(
-                [
-                    {"role": "user", "content": user},
-                    {"role": "assistant", "content": assistant},
-                ]
-            ) 
-        conversation.append({"role": "user", "content": f"Contesto: {context}\n\n Nuova domanda: {message} . Rispondi in italiano e seguendo i comandi che ti ho dato prima"})
-        print("debug - CONV3", conversation)
-        
-        """ retriever = db.as_retriever()
-        qa = RetrievalQA.from_chain_type(llm=model, chain_type="refine", retriever=retriever, return_source_documents=False)
-        question = "Cosa sono i RWA?"
-        result = qa.run({"query": question})
-        print(result["result"]) """
-
-    # Iterate model output
-    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
-    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
-        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
-        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
-    input_ids = input_ids.to(model.device)
-
-    streamer = TextIteratorStreamer(tokenizer, timeout=None, skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        {"input_ids": input_ids},
-        streamer=streamer,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        top_p=top_p,
-        top_k=top_k,
-        temperature=temperature,
-        num_beams=1,
-        repetition_penalty=repetition_penalty,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-    
-
-    outputs = []
-    for text in streamer:
-        outputs.append(text)
-        yield "".join(outputs)
-    
-    if session_state["documents_loaded"]:
-        outputs.append(f"Fonti utilizzate: {sources}")
-        yield "".join(outputs) 
-        
-    #sources = []    
-    print("debug - CHATHISTORY", chat_history)    
-
-chat_interface = gr.ChatInterface(
-    fn=generate,
-    additional_inputs=[
-        gr.Slider(
-            label="Max new tokens",
-            minimum=1,
-            maximum=MAX_MAX_NEW_TOKENS,
-            step=1,
-            value=DEFAULT_MAX_NEW_TOKENS,
-        ),
-        gr.Slider(
-            label="Temperature",
-            minimum=0.1,
-            maximum=4.0,
-            step=0.1,
-            value=0.6,
-        ),
-        gr.Slider(
-            label="Top-p (nucleus sampling)",
-            minimum=0.05,
-            maximum=1.0,
-            step=0.05,
-            value=0.9,
-        ),
-        gr.Slider(
-            label="Top-k",
-            minimum=1,
-            maximum=1000,
-            step=1,
-            value=50,
-        ),
-        gr.Slider(
-            label="Repetition penalty",
-            minimum=1.0,
-            maximum=2.0,
-            step=0.05,
-            value=1.2,
-        ),
-    ],
-    stop_btn=None,
-    examples=[
-        ["Ciao, in cosa puoi aiutarmi?"],
-        ["Ciao, in cosa consiste un piatto di spaghetti?"],
-        ["Ciao, quali sono le aziende che hanno iniziato ad integrare le stablecoins? Fammi un breve sommario."],
-        ["Spiegami la differenza tra mondi virtuali pubblici o privati"],
-        ["Trovami un esempio di progetto B2B"],
-        ["Quali sono le regole europee sui bonifici istantanei?"],
-    ],
-    cache_examples=False,
-)
-
-with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}", fill_height=True) as demo:
-    gr.Markdown(DESCRIPTION, elem_classes="centered")
-    chat_interface.render()
-
-if __name__ == "__main__":
-    #demo.queue(max_size=20).launch()
-    demo.launch()

interface.py

@@ -1,105 +0,0 @@
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from llama_index.core.llms import  CustomLLM, LLMMetadata, CompletionResponse, CompletionResponseGen
-from llama_index.core.llms.callbacks import llm_completion_callback
-from typing import Any, Iterator
-import torch
-from transformers import TextIteratorStreamer
-from threading import Thread
-from pydantic import Field, field_validator
-import keras
-import keras_nlp
-
-# for transformers 2 (__setattr__ is used to bypass Pydantic check )
-"""class GemmaLLMInterface(CustomLLM):
-    def __init__(self, model_id: str = "google/gemma-2-2b-it", **kwargs):
-        super().__init__(**kwargs)
-        object.__setattr__(self, "model_id", model_id)  
-        model = AutoModelForCausalLM.from_pretrained(
-            model_id,
-            device_map="auto",
-            torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
-        )
-        tokenizer = AutoTokenizer.from_pretrained(model_id)
-        object.__setattr__(self, "model", model)  
-        object.__setattr__(self, "tokenizer", tokenizer) 
-        object.__setattr__(self, "context_window", 8192)
-        object.__setattr__(self, "num_output", 2048)
-
-    def _format_prompt(self, message: str) -> str:
-        return (
-            f"<start_of_turn>user\n{message}<end_of_turn>\n"
-            f"<start_of_turn>model\n"
-        )
-
-    @property
-    def metadata(self) -> LLMMetadata:
-        return LLMMetadata(
-            context_window=self.context_window,
-            num_output=self.num_output,
-            model_name=self.model_id, 
-        )
-        
-        
-    @llm_completion_callback()
-    def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
-        prompt = self._format_prompt(prompt)
-        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
-        outputs = self.model.generate(**inputs, max_new_tokens=self.num_output)
-        response = self.tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
-        response = response[len(prompt):].strip()
-        return CompletionResponse(text=response if response else "No response generated.")
-
-    @llm_completion_callback()
-    def stream_complete(self, prompt: str, **kwargs: Any) -> CompletionResponseGen:
-        #prompt = self._format_prompt(prompt)
-        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
-        
-        streamer = TextIteratorStreamer(self.tokenizer, skip_special_tokens=True)
-        generation_kwargs = dict(inputs, max_new_tokens=self.num_output, streamer=streamer)
-        
-        thread = Thread(target=self.model.generate, kwargs=generation_kwargs)
-        thread.start()
-
-        streamed_response = ""
-        for new_text in streamer:
-            if new_text:
-                streamed_response += new_text
-                yield CompletionResponse(text=streamed_response, delta=new_text)
-        
-        if not streamed_response:
-            yield CompletionResponse(text="No response generated.", delta="No response generated.")"""
-
-# for Keras
-class GemmaLLMInterface(CustomLLM):
-    model: keras_nlp.models.GemmaCausalLM = None
-    context_window: int = 8192
-    num_output: int = 2048
-    model_name: str = "gemma_2"
-
-    def _format_prompt(self, message: str) -> str:
-        return (
-            f"<start_of_turn>user\n{message}<end_of_turn>\n" f"<start_of_turn>model\n"
-        )
-
-    @property
-    def metadata(self) -> LLMMetadata:
-        """Get LLM metadata."""
-        return LLMMetadata(
-            context_window=self.context_window,
-            num_output=self.num_output,
-            model_name=self.model_name,
-        )
-
-    @llm_completion_callback()
-    def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
-        prompt = self._format_prompt(prompt)
-        raw_response = self.model.generate(prompt, max_length=self.num_output)
-        response = raw_response[len(prompt) :]
-        return CompletionResponse(text=response)
-
-    @llm_completion_callback()
-    def stream_complete(self, prompt: str, **kwargs: Any) -> CompletionResponseGen:
-        response = self.complete(prompt).text
-        for token in response:
-            response += token
-            yield CompletionResponse(text=response, delta=token)

requirements.txt

@@ -14,6 +14,7 @@ torch==2.2.0
 transformers==4.43.3
 llama-cpp-agent>=0.2.25
 setuptools
+faiss-cpu
 
 pydantic
 ipython