Fine-tuned all-mpnet-base-v2 for SCION RAG retrieval

README.md

@@ -375,49 +375,49 @@ model-index:
       type: val-ir-eval
     metrics:
     - type: cosine_accuracy@1
-      value: 0.6153653653653653
+      value: 0.6293793793793794
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.8033033033033034
+      value: 0.8215715715715716
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.8578578578578578
+      value: 0.8763763763763763
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.9179179179179179
+      value: 0.9309309309309309
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.6153653653653653
+      value: 0.6293793793793794
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.268018018018018
+      value: 0.2739406072739406
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.17182182182182182
+      value: 0.17547547547547548
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.091991991991992
+      value: 0.09334334334334335
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.6151290179067957
+      value: 0.6291916916916916
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.8029696363029697
+      value: 0.8209737515293072
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.8575519964408853
+      value: 0.8758689244800356
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.9174660771882994
+      value: 0.9305555555555556
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.7686494924105739
+      value: 0.7827567470448342
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.7208215159604052
+      value: 0.7351305670750117
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.7240690909632143
+      value: 0.7379411341051004
       name: Cosine Map@100
 ---
 
@@ -520,21 +520,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | Value      |
 |:--------------------|:-----------|
-| cosine_accuracy@1   | 0.6154     |
-| cosine_accuracy@3   | 0.8033     |
-| cosine_accuracy@5   | 0.8579     |
-| cosine_accuracy@10  | 0.9179     |
-| cosine_precision@1  | 0.6154     |
-| cosine_precision@3  | 0.268      |
-| cosine_precision@5  | 0.1718     |
-| cosine_precision@10 | 0.092      |
-| cosine_recall@1     | 0.6151     |
-| cosine_recall@3     | 0.803      |
-| cosine_recall@5     | 0.8576     |
-| cosine_recall@10    | 0.9175     |
-| **cosine_ndcg@10**  | **0.7686** |
-| cosine_mrr@10       | 0.7208     |
-| cosine_map@100      | 0.7241     |
+| cosine_accuracy@1   | 0.6294     |
+| cosine_accuracy@3   | 0.8216     |
+| cosine_accuracy@5   | 0.8764     |
+| cosine_accuracy@10  | 0.9309     |
+| cosine_precision@1  | 0.6294     |
+| cosine_precision@3  | 0.2739     |
+| cosine_precision@5  | 0.1755     |
+| cosine_precision@10 | 0.0933     |
+| cosine_recall@1     | 0.6292     |
+| cosine_recall@3     | 0.821      |
+| cosine_recall@5     | 0.8759     |
+| cosine_recall@10    | 0.9306     |
+| **cosine_ndcg@10**  | **0.7828** |
+| cosine_mrr@10       | 0.7351     |
+| cosine_map@100      | 0.7379     |
 
 <!--
 ## Bias, Risks and Limitations
@@ -581,7 +581,6 @@ You can finetune this model on your own dataset.
 - `eval_strategy`: steps
 - `per_device_train_batch_size`: 64
 - `per_device_eval_batch_size`: 64
-- `num_train_epochs`: 1
 - `fp16`: True
 - `multi_dataset_batch_sampler`: round_robin
 
@@ -605,7 +604,7 @@ You can finetune this model on your own dataset.
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
 - `max_grad_norm`: 1
-- `num_train_epochs`: 1
+- `num_train_epochs`: 3
 - `max_steps`: -1
 - `lr_scheduler_type`: linear
 - `lr_scheduler_kwargs`: {}
@@ -707,12 +706,30 @@ You can finetune this model on your own dataset.
 </details>
 
 ### Training Logs
-| Epoch  | Step | val-ir-eval_cosine_ndcg@10 |
-|:------:|:----:|:--------------------------:|
-| 0.2740 | 100  | 0.7363                     |
-| 0.5479 | 200  | 0.7595                     |
-| 0.8219 | 300  | 0.7648                     |
-| 1.0    | 365  | 0.7686                     |
+| Epoch  | Step | Training Loss | val-ir-eval_cosine_ndcg@10 |
+|:------:|:----:|:-------------:|:--------------------------:|
+| 0.1372 | 100  | -             | 0.6950                     |
+| 0.2743 | 200  | -             | 0.7313                     |
+| 0.4115 | 300  | -             | 0.7443                     |
+| 0.5487 | 400  | -             | 0.7573                     |
+| 0.6859 | 500  | 0.3862        | 0.7576                     |
+| 0.8230 | 600  | -             | 0.7627                     |
+| 0.9602 | 700  | -             | 0.7662                     |
+| 1.0    | 729  | -             | 0.7709                     |
+| 1.0974 | 800  | -             | 0.7705                     |
+| 1.2346 | 900  | -             | 0.7718                     |
+| 1.3717 | 1000 | 0.2356        | 0.7747                     |
+| 1.5089 | 1100 | -             | 0.7742                     |
+| 1.6461 | 1200 | -             | 0.7759                     |
+| 1.7833 | 1300 | -             | 0.7776                     |
+| 1.9204 | 1400 | -             | 0.7807                     |
+| 2.0    | 1458 | -             | 0.7815                     |
+| 2.0576 | 1500 | 0.1937        | 0.7789                     |
+| 2.1948 | 1600 | -             | 0.7814                     |
+| 2.3320 | 1700 | -             | 0.7819                     |
+| 2.4691 | 1800 | -             | 0.7823                     |
+| 2.6063 | 1900 | -             | 0.7827                     |
+| 2.7435 | 2000 | 0.1758        | 0.7828                     |
 
 
 ### Framework Versions

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cb83b318d23c5cd6330c6961171323226ef730efec1b139f7d26c5150c2a9c4
+oid sha256:547f3edd35f12fd139aa0b67815aae00adc942a48478cb8352cbb49ea259f697
 size 90864192