pdudka commited on
Commit
6cdd644
·
1 Parent(s): e9d3c4d

troubleshooting vectorstore stuff

Browse files
Files changed (1) hide show
  1. app.py +7 -7
app.py CHANGED
@@ -61,9 +61,9 @@ hf_embeddings = HuggingFaceEndpointEmbeddings(
61
 
62
  ### 4. INDEX FILES
63
  ### NOTE: REMEMBER TO BATCH THE DOCUMENTS WITH MAXIMUM BATCH SIZE = 32
64
- if os.path.exists("./data/vectorstore"):
65
  vectorstore = FAISS.load_local(
66
- "./data/vectorstore",
67
  hf_embeddings,
68
  allow_dangerous_deserialization=True # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
69
  )
@@ -71,13 +71,13 @@ if os.path.exists("./data/vectorstore"):
71
  print("Loaded Vectorstore")
72
  else:
73
  print("Indexing Files")
74
- os.makedirs("./data/vectorstore", exist_ok=True)
75
- for i in range(0, len(split_documents), 32):
76
  if i == 0:
77
- vectorstore = FAISS.from_documents(split_documents[i:i+32], hf_embeddings)
78
  continue
79
- vectorstore.add_documents(split_documents[i:i+32])
80
- vectorstore.save_local("./data/vectorstore")
81
 
82
  hf_retriever = vectorstore.as_retriever()
83
 
 
61
 
62
  ### 4. INDEX FILES
63
  ### NOTE: REMEMBER TO BATCH THE DOCUMENTS WITH MAXIMUM BATCH SIZE = 32
64
+ if os.path.exists(VECTORSTORE_PATH):
65
  vectorstore = FAISS.load_local(
66
+ VECTORSTORE_DIR,#"./data/vectorstore",
67
  hf_embeddings,
68
  allow_dangerous_deserialization=True # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
69
  )
 
71
  print("Loaded Vectorstore")
72
  else:
73
  print("Indexing Files")
74
+ os.makedirs(VECTORSTORE_DIR, exist_ok=True)
75
+ for i in range(0, len(split_chunks), 32):
76
  if i == 0:
77
+ vectorstore = FAISS.from_documents(split_chunks[i:i+32], hf_embeddings)
78
  continue
79
+ vectorstore.add_documents(split_chunks[i:i+32])
80
+ vectorstore.save_local(VECTORSTORE_DIR)
81
 
82
  hf_retriever = vectorstore.as_retriever()
83