vectostoreindex

app.py

@@ -42,9 +42,6 @@ model.config.sliding_window = 4096
 #model = model.to(device)
 model.eval()
 
-Settings.embed_model = InstructorEmbedding(model_name="hkunlp/instructor-base")
-
-
 ###------####
 # rag
 documents_paths = {
@@ -64,14 +61,14 @@ INSTRUCTION_1 = 'In italiano, chiedi sempre se la domanda si riferisce agli "Oss
 INSTRUCTION_2 = 'Sei un assistente che risponde sempre in italiano alle domande basandosi solo sulle informazioni fornite nel contesto che ti darò. Se non trovi informazioni, rispondi "Puoi chiedere maggiori informazioni all\'ufficio di riferimento.". Se invece la domanda è completamente fuori contesto, non rispondere e rammenta il topic del contesto'
 
 
-# Reading documents from disk
+"""# Reading documents from disk
 docs = SimpleDirectoryReader(input_files=["data/blockchainprova.txt"]).load_data()
             # Splitting the document into chunks with
             # predefined size and overlap
 parser = SentenceSplitter.from_defaults(
                 chunk_size=256, chunk_overlap=64, paragraph_separator="\n\n"
             )
-nodes = parser.get_nodes_from_documents(docs)
+nodes = parser.get_nodes_from_documents(docs)"""
 
 
 @spaces.GPU()
@@ -117,7 +114,7 @@ def generate(
 
             ######
             
-            index = VectorStoreIndex(nodes)
+            """index = VectorStoreIndex(nodes)
             # get retriver
             retriever = index.as_retriever(similarity_top_k=3)
             relevant_chunks = retriever.retrieve(message)
@@ -126,7 +123,7 @@ def generate(
                 
                 info_message += f"{idx + 1}) {chunk.text[:64]}...\n"
                 print(info_message)
-                gr.Info(info_message)
+                gr.Info(info_message)"""
             
             session_state["index"] = True
             

backend.py

@@ -16,6 +16,7 @@ from llama_index.core.llms import ChatMessage, MessageRole , CompletionResponse
 from IPython.display import Markdown, display
 #from langchain.embeddings.huggingface import HuggingFaceEmbeddings
 #from llama_index import LangchainEmbedding, ServiceContext
+from llama_index.llms.huggingface import HuggingFaceLLM
 
 
 
@@ -38,8 +39,35 @@ model.eval()"""
 Settings.embed_model = InstructorEmbedding(model_name="hkunlp/instructor-base")
 #Settings.embed_model = LangchainEmbedding(HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2'))
 
+SYSTEM_PROMPT = """You are an AI assistant that answers questions in a friendly manner, based on the given source documents. Here are some rules you always follow:
+- Generate human readable output, avoid creating output with gibberish text.
+- Generate only the requested output, don't include any other language before or after the requested output.
+- Never say thank you, that you are happy to help, that you are an AI agent, etc. Just answer directly.
+- Generate professional language typically used in business documents in North America.
+- Never generate offensive or foul language.
+"""
+
+query_wrapper_prompt = PromptTemplate(
+    "[INST]<<SYS>>\n" + SYSTEM_PROMPT + "<</SYS>>\n\n{query_str}[/INST] "
+)
+
+
+model_id = "google/gemma-2-2b-it"
+llm = HuggingFaceLLM(
+    context_window=4096,
+    max_new_tokens=2048,
+    generate_kwargs={"temperature": 0.0, "do_sample": False},
+    query_wrapper_prompt=query_wrapper_prompt,
+    tokenizer_name=model_id,
+    model_name=model_id,
+    device_map="auto",
+    # change these settings below depending on your GPU
+    model_kwargs={"torch_dtype": torch.float16, "load_in_8bit": True},
+)
 
 Settings.llm = GemmaLLMInterface()
+Settings.llm = llm
+
 
 documents_paths = {
     'blockchain': 'data/blockchainprova.txt',