Spaces:
Runtime error
Runtime error
troubleshooting vectorstore stuff
Browse files
app.py
CHANGED
@@ -61,9 +61,9 @@ hf_embeddings = HuggingFaceEndpointEmbeddings(
|
|
61 |
|
62 |
### 4. INDEX FILES
|
63 |
### NOTE: REMEMBER TO BATCH THE DOCUMENTS WITH MAXIMUM BATCH SIZE = 32
|
64 |
-
if os.path.exists(
|
65 |
vectorstore = FAISS.load_local(
|
66 |
-
"./data/vectorstore",
|
67 |
hf_embeddings,
|
68 |
allow_dangerous_deserialization=True # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
|
69 |
)
|
@@ -71,13 +71,13 @@ if os.path.exists("./data/vectorstore"):
|
|
71 |
print("Loaded Vectorstore")
|
72 |
else:
|
73 |
print("Indexing Files")
|
74 |
-
os.makedirs(
|
75 |
-
for i in range(0, len(
|
76 |
if i == 0:
|
77 |
-
vectorstore = FAISS.from_documents(
|
78 |
continue
|
79 |
-
vectorstore.add_documents(
|
80 |
-
vectorstore.save_local(
|
81 |
|
82 |
hf_retriever = vectorstore.as_retriever()
|
83 |
|
|
|
61 |
|
62 |
### 4. INDEX FILES
|
63 |
### NOTE: REMEMBER TO BATCH THE DOCUMENTS WITH MAXIMUM BATCH SIZE = 32
|
64 |
+
if os.path.exists(VECTORSTORE_PATH):
|
65 |
vectorstore = FAISS.load_local(
|
66 |
+
VECTORSTORE_DIR,#"./data/vectorstore",
|
67 |
hf_embeddings,
|
68 |
allow_dangerous_deserialization=True # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
|
69 |
)
|
|
|
71 |
print("Loaded Vectorstore")
|
72 |
else:
|
73 |
print("Indexing Files")
|
74 |
+
os.makedirs(VECTORSTORE_DIR, exist_ok=True)
|
75 |
+
for i in range(0, len(split_chunks), 32):
|
76 |
if i == 0:
|
77 |
+
vectorstore = FAISS.from_documents(split_chunks[i:i+32], hf_embeddings)
|
78 |
continue
|
79 |
+
vectorstore.add_documents(split_chunks[i:i+32])
|
80 |
+
vectorstore.save_local(VECTORSTORE_DIR)
|
81 |
|
82 |
hf_retriever = vectorstore.as_retriever()
|
83 |
|