Update app.py

app.py

@@ -5,11 +5,14 @@
 import streamlit as st
 import requests
 from bs4 import BeautifulSoup
-from langchain.document_loaders import TextLoader          #reads in a file as text and places it all into one document.
-from langchain.indexes import VectorstoreIndexCreator      #Logic for creating indexes.
-from langchain.vectorstores import DocArrayInMemorySearch  #document index provided by Docarray that stores documents in memory.
+#from langchain.indexes import VectorstoreIndexCreator      #Logic for creating indexes.
+#from langchain.vectorstores import DocArrayInMemorySearch  #document index provided by Docarray that stores documents in memory.
 from sentence_transformers import SentenceTransformer
 from langchain_community.llms import HuggingFaceEndpoint
+from langchain_chroma import Chroma
+from langchain_community.document_loaders import TextLoader
+from langchain_community.embeddings.sentence_transformer import (SentenceTransformerEmbeddings,)
+from langchain_text_splitters import CharacterTextSplitter
 
 #import vertexai
 #from langchain.llms import VertexAI
@@ -52,11 +55,21 @@ def create_langchain_index(input_text):
     print("--indexing---")
     get_text(input_text)
     loader = TextLoader("text\\temp.txt", encoding='utf-8')
-    data = loader.load()
-    data = str(data)
-    embeddings = model.encode(data)
-    index = VectorstoreIndexCreator(vectorstore_cls=DocArrayInMemorySearch,embedding=embeddings).from_loaders([loader])
-    return index
+    documents = loader.load()
+    # split it into chunks
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    docs = text_splitter.split_documents(documents)
+    print(docs)
+    # create the open-source embedding function
+    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    # load it into Chroma
+    db = Chroma.from_documents(docs, embeddings)
+    persist_directory = "chroma_db"  
+    vectordb = Chroma.from_documents(
+        documents=docs, embedding=embeddings, persist_directory=persist_directory
+    )
+    new_db = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+    return new_db
 
 # @st.cache_resource
 # def get_basic_page_details(input_text,summary_query,tweet_query,ln_query):