Upload retriever to Hugging Face Hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+database.lmdb/data.mdb filter=lfs diff=lfs merge=lfs -text
+indexes/bm25/vocab.index.json filter=lfs diff=lfs merge=lfs -text
+indexes/contriever/index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,101 @@
+---
+language: en
+library_name: FlexRAG
+tags:
+- FlexRAG
+- retrieval
+- search
+- lexical
+- RAG
+- IR
+---
+
+# FlexRAG Retriever
+
+This is a FlexRetriever created with the [`FlexRAG`](https://github.com/ictnlp/FlexRAG) library (version `0.3.0`).
+
+## Retriever Attributes
+The `enwiki_2020_atlas` retriever is a FlexRetriever that provides access to the English Wikipedia corpus from December 2020. It is designed for information retrieval tasks, allowing users to search and retrieve relevant documents based on their queries.
+The corpus of this retriever was created by the [Atlas](https://github.com/facebookresearch/atlas) project and the index was built using the [FlexRAG](https://github.com/ictnlp/FlexRAG) library.
+
+| Corpus Attribute | Value                                                           |
+| ---------------- | --------------------------------------------------------------- |
+| Language         | English                                                         |
+| Domain           | Wikipedia                                                       |
+| Saved Fields     | title, section, text                                            |
+| Size             | 33.1M (29.4M text, 3.8M infobox)                                |
+| Dump Date        | Dec 2020                                                        |
+| Provideer        | [Atlas](https://github.com/facebookresearch/atlas)              |
+| License          | [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) |
+
+
+| Index Attribute | Value                                                           |
+| --------------- | --------------------------------------------------------------- |
+| Index Name      | bm25                                                            |
+| Index Type      | Sparse                                                          |
+| Index Method    | Lucene                                                          |
+| Indexed Fields  | title, section, text (concat)                                   |
+| Preprocessing   | LengthFilter(min_char=10, max_char=4096)                        |
+| Provideer       | [FlexRAG](https://github.com/ictnlp/flexrag)                    |
+| License         | [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) |
+
+| Index Attribute | Value                                                           |
+| --------------- | --------------------------------------------------------------- |
+| Index Name      | contriever                                                      |
+| Index Type      | Dense                                                           |
+| Index Method    | IVFPQ                                                           |
+| Indexed Fields  | title, section, text (concat)                                   |
+| Query Encoder   | `facebook/contriever-msmarco`                                   |
+| Passage Encoder | `facebook/contriever-msmarco`                                   |
+| Preprocessing   | LengthFilter(min_char=10, max_char=4096)                        |
+| Provideer       | [FlexRAG](https://github.com/ictnlp/flexrag)                    |
+| License         | [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) |
+
+## Usage
+
+### Installation
+You can install the `FlexRAG` library with `pip`:
+
+```bash
+pip install flexrag faiss-cpu
+```
+
+### Loading the `FlexRAG` retriever
+
+You can use this retriever for information retrieval tasks. Here is an example:
+
+```python
+from flexrag.retriever import LocalRetriever
+
+
+# Load the retriever from the HuggingFace Hub
+retriever = LocalRetriever.load_from_hub("FlexRAG/enwiki_2020_atlas")
+
+
+# You can retrieve relevant documents now
+results = retriever.search("Who is Bruce Wayne?")
+```
+
+### Running the RAG demo with the retriever
+
+You can run the **GUI application** of the RAG assistant with this retriever. Here is an example:
+
+```bash
+python -m flexrag.entrypoints.run_interactive \
+    assistant_type=modular \
+    modular_config.used_fields=[title,text] \
+    modular_config.retriever_type="FlexRAG/enwiki_2020_atlas" \
+    modular_config.response_type=original \
+    modular_config.generator_type=openai \
+    modular_config.openai_config.model_name='gpt-4o-mini' \
+    modular_config.openai_config.api_key=$OPENAI_KEY \
+    modular_config.do_sample=False
+```
+
+## License
+As the corpus is based on the [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) license, the retriever is also licensed under the same license.
+
+
+FlexRAG Related Links:
+* 📚[Documentation](https://flexrag.readthedocs.io/en/latest/)
+* 💻[GitHub Repository](https://github.com/ictnlp/flexrag)

cls.id

@@ -0,0 +1 @@
+FlexRetriever

config.yaml

@@ -0,0 +1,37 @@
+log_interval: 100000
+top_k: 10
+batch_size: 4096
+query_preprocess_pipeline:
+  processor_type: []
+  length_filter_config:
+    max_tokens: null
+    min_tokens: null
+    max_chars: null
+    min_chars: null
+    max_bytes: null
+    min_bytes: null
+    tokenizer_config:
+      tokenizer_type: moses
+      hf_tokenizer_path: null
+      tiktok_tokenizer_name: null
+      lang: null
+  token_normalize_config:
+    lang: en
+    penn: true
+    norm_quote_commas: true
+    norm_numbers: true
+    pre_replace_unicode_punct: false
+    post_remove_control_chars: false
+    perl_parity: false
+  truncate_config:
+    max_chars: null
+    max_bytes: null
+    max_tokens: null
+    tokenizer_config:
+      tokenizer_type: moses
+      hf_tokenizer_path: null
+      tiktok_tokenizer_name: null
+      lang: null
+retriever_path: null
+indexes_merge_method: linear
+used_indexes: null

database.lmdb/data.mdb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e5065b6ccae1bd8fe3f1755183ece35b474acc43c651b0f0171cca166f9715f
+size 31990173696

indexes/bm25/cls.id

@@ -0,0 +1 @@
+BM25Index

indexes/bm25/config.yaml

@@ -0,0 +1,10 @@
+log_interval: 10000
+batch_size: 512
+index_path: /data/zhangzhuocheng/Lab/Python/LLM/datasets/RAG/Corpus/enwiki_2020_atlas/flex/indexes/bm25
+method: lucene
+idf_method: null
+backend: auto
+k1: 1.5
+b: 0.75
+delta: 0.5
+lang: english

indexes/bm25/context_mapping.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c0cd52bb5be194a99594ca416f6b55846e1d3e98f02bde2c3fc83442ef3559c
+size 1046861765

indexes/bm25/data.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd881d5b05243218044350a4ffcfc6a04529b02bfc2825fb84539510e23eb94f
+size 7344164228

indexes/bm25/indices.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06568137080de8c414bec72a29673b5b073dca1f5d71bc1d3acc82b65e941463
+size 7344164228

indexes/bm25/indptr.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d717c1e86b0299ebf46cc692346272bae97dd350967993f979461d550634473c
+size 45940392

indexes/bm25/multi_field_index_config.yaml

@@ -0,0 +1,5 @@
+indexed_fields:
+- title
+- section
+- text
+merge_method: concat

indexes/bm25/params.index.json

@@ -0,0 +1,12 @@
+{
+    "k1": 1.5,
+    "b": 0.75,
+    "delta": 0.5,
+    "method": "lucene",
+    "idf_method": "lucene",
+    "dtype": "float32",
+    "int_dtype": "int32",
+    "num_docs": 35528846,
+    "version": "0.2.11",
+    "backend": "numpy"
+}

indexes/bm25/vocab.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c15bb4c04602797055fe19a90ba6cea2c08bc96990495d84c3d1d1536682bc7
+size 235687356

indexes/contriever/cls.id

@@ -0,0 +1 @@
+FaissIndex

indexes/contriever/config.yaml

@@ -0,0 +1,62 @@
+log_interval: 100000
+batch_size: 2048
+index_path: null
+query_encoder_config:
+  encoder_type: hf
+  cohere_config: null
+  hf_config:
+    batch_size: 32
+    log_interval: 1000
+    model_path: facebook/contriever-msmarco
+    tokenizer_path: null
+    trust_remote_code: false
+    device_id:
+    - 0
+    load_dtype: auto
+    max_encode_length: 512
+    encode_method: mean
+    normalize: false
+    prompt: ''
+    task: ''
+  hf_clip_config: null
+  jina_config: null
+  ollama_config: null
+  openai_config: null
+  sentence_transformer_config: null
+passage_encoder_config:
+  encoder_type: hf
+  cohere_config: null
+  hf_config:
+    batch_size: 32
+    log_interval: 1000
+    model_path: facebook/contriever-msmarco
+    tokenizer_path: null
+    trust_remote_code: false
+    device_id:
+    - 0
+    - 1
+    - 2
+    - 3
+    load_dtype: auto
+    max_encode_length: 512
+    encode_method: mean
+    normalize: false
+    prompt: ''
+    task: ''
+  hf_clip_config: null
+  jina_config: null
+  ollama_config: null
+  openai_config: null
+  sentence_transformer_config: null
+distance_function: IP
+index_type: auto
+n_subquantizers: 8
+n_bits: 8
+n_list: 1000
+factory_str: null
+index_train_num: -1
+n_probe: 512
+device_id: []
+k_factor: 10
+polysemous_ht: 0
+efSearch: 100

indexes/contriever/context_mapping.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c0cd52bb5be194a99594ca416f6b55846e1d3e98f02bde2c3fc83442ef3559c
+size 1046861765

indexes/contriever/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eb013a80c97be41a945ccc02f887651c7c135ab6f65bb0551c4a2e8818c8fea
+size 7130740032

indexes/contriever/multi_field_index_config.yaml

@@ -0,0 +1,5 @@
+indexed_fields:
+- title
+- section
+- text
+merge_method: concat