Spaces:
Sleeping
Sleeping
Commit
·
ed0c3ad
1
Parent(s):
5ab83b7
- app.py +20 -22
- faiss_index +0 -0
- faiss_index.json +0 -1
- inmemory_document_store.pkl +3 -0
- requirements.txt +3 -3
- responses.json +0 -0
app.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
| 1 |
import json
|
|
|
|
| 2 |
import streamlit as st
|
| 3 |
-
from haystack.document_stores import FAISSDocumentStore
|
| 4 |
-
|
| 5 |
from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
|
| 6 |
from haystack.nodes import DensePassageRetriever
|
| 7 |
from haystack.nodes import FARMReader
|
|
@@ -22,11 +23,12 @@ st.title("DPR on Supreme Court Judgements (Capital Gain)")
|
|
| 22 |
# } for doc in data
|
| 23 |
# ]
|
| 24 |
|
| 25 |
-
# document_store = FAISSDocumentStore(embedding_dim=768, faiss_index_factory_str="Flat")
|
| 26 |
-
|
|
|
|
| 27 |
# document_store.write_documents(documents)
|
| 28 |
|
| 29 |
-
document_store = FAISSDocumentStore.load("faiss_index")
|
| 30 |
|
| 31 |
retriever = DensePassageRetriever(
|
| 32 |
document_store=document_store,
|
|
@@ -35,29 +37,25 @@ retriever = DensePassageRetriever(
|
|
| 35 |
)
|
| 36 |
|
| 37 |
# document_store.update_embeddings(retriever)
|
| 38 |
-
# document_store.save("./
|
| 39 |
-
|
|
|
|
| 40 |
|
| 41 |
|
| 42 |
reader = FARMReader(model_name_or_path="deepset/bert-base-cased-squad2")
|
| 43 |
|
| 44 |
pipeline = ExtractiveQAPipeline(reader=reader, retriever=retriever)
|
| 45 |
|
| 46 |
-
|
| 47 |
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
# # st.write(f"**{idx + 1}. {result.meta['name']}**")
|
| 54 |
-
# # st.write(f"URL: {result.meta['url']}")
|
| 55 |
-
# # st.write(result.content)
|
| 56 |
-
# # st.write("---")
|
| 57 |
|
| 58 |
# query = st.text_input("Enter Question")
|
| 59 |
-
query = "What is the subject matter of the petition in the Sadanand S. Varde case?"
|
| 60 |
-
result = pipeline.run(query=query, params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}})
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
st.markdown(f"=====================\nAnswer: {answer.answer}\nContext: {answer.context}\nScore: {answer.score}")
|
|
|
|
| 1 |
import json
|
| 2 |
+
import pickle
|
| 3 |
import streamlit as st
|
| 4 |
+
# from haystack.document_stores import FAISSDocumentStore
|
| 5 |
+
from haystack.document_stores import InMemoryDocumentStore
|
| 6 |
from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
|
| 7 |
from haystack.nodes import DensePassageRetriever
|
| 8 |
from haystack.nodes import FARMReader
|
|
|
|
| 23 |
# } for doc in data
|
| 24 |
# ]
|
| 25 |
|
| 26 |
+
# document_store = FAISSDocumentStore(embedding_dim=768, faiss_index_factory_str="Flat", sql_url="sqlite:///faiss_document_store.d")
|
| 27 |
+
with open("inmemory_document_store.pkl", "rb") as f:
|
| 28 |
+
document_store = pickle.load(f)
|
| 29 |
# document_store.write_documents(documents)
|
| 30 |
|
| 31 |
+
# document_store = FAISSDocumentStore.load(index_path="./faiss_index", config_path="./faiss_index.json")
|
| 32 |
|
| 33 |
retriever = DensePassageRetriever(
|
| 34 |
document_store=document_store,
|
|
|
|
| 37 |
)
|
| 38 |
|
| 39 |
# document_store.update_embeddings(retriever)
|
| 40 |
+
# document_store.save(index_path="./faiss_index", config_path="./faiss_index.json")
|
| 41 |
+
# with open("inmemory_document_store.pkl", "wb") as f:
|
| 42 |
+
# pickle.dump(document_store, f)
|
| 43 |
|
| 44 |
|
| 45 |
reader = FARMReader(model_name_or_path="deepset/bert-base-cased-squad2")
|
| 46 |
|
| 47 |
pipeline = ExtractiveQAPipeline(reader=reader, retriever=retriever)
|
| 48 |
|
| 49 |
+
query = st.text_input("Enter your query:", "")
|
| 50 |
|
| 51 |
+
if query:
|
| 52 |
+
with st.spinner("Searching..."):
|
| 53 |
+
results = pipeline.run(query=query, params={"Retriever": {"top_k": 5}})
|
| 54 |
+
for answer in results['answers']:
|
| 55 |
+
st.markdown(f"=====================\nAnswer: {answer.answer}\nContext: {answer.context}\nScore: {answer.score}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
# query = st.text_input("Enter Question")
|
| 58 |
+
# query = "What is the subject matter of the petition in the Sadanand S. Varde case?"
|
| 59 |
+
# result = pipeline.run(query=query, params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}})
|
| 60 |
+
# for answer in result['answers']:
|
| 61 |
+
# print(f"=====================\nAnswer: {answer.answer}\nContext: {answer.context}\nScore: {answer.score}")
|
|
|
faiss_index
DELETED
|
Binary file (338 kB)
|
|
|
faiss_index.json
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
{"embedding_dim": 768, "faiss_index_factory_str": "Flat"}
|
|
|
|
|
|
inmemory_document_store.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc70075a2b22abd70770753946ba868c925afbb370760210b8d38cb87cc132c8
|
| 3 |
+
size 6070663
|
requirements.txt
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
farm-haystack
|
| 2 |
-
transformers
|
|
|
|
| 3 |
faiss-cpu
|
| 4 |
farm-haystack[inference]
|
| 5 |
farm-haystack[faiss]
|
| 6 |
-
sentence-transformers
|
| 7 |
-
pydantic==1.10.14
|
|
|
|
| 1 |
farm-haystack
|
| 2 |
+
transformers
|
| 3 |
+
streamlit
|
| 4 |
faiss-cpu
|
| 5 |
farm-haystack[inference]
|
| 6 |
farm-haystack[faiss]
|
| 7 |
+
sentence-transformers
|
|
|
responses.json
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|