Updated to 24 models. 10 models outperform the current SOTA

add_new_model/mteb_metadata.yaml

@@ -2,7 +2,7 @@
 tags:
 - mteb
 model-index:
-- name: multilingual-e5-base-stsb-tuned-b16-e10-all-languages
+- name: multilingual-e5-large-STSAUGMENTED-b16-e10
   results:
   - task:
       type: Classification
@@ -14,9 +14,9 @@ model-index:
       revision: 1399c76144fd37290681b995c656ef9b2e06e26d
     metrics:
     - type: accuracy
-      value: 43.04
+      value: 42.114
     - type: f1
-      value: 40.906126104515174
+      value: 40.962817058318876
   - task:
       type: Classification
     dataset:
@@ -27,9 +27,9 @@ model-index:
       revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
     metrics:
     - type: accuracy
-      value: 88.81254169446298
+      value: 80.92394929953302
     - type: f1
-      value: 88.19994853608793
+      value: 80.87019762034018
   - task:
       type: Classification
     dataset:
@@ -40,9 +40,9 @@ model-index:
       revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
     metrics:
     - type: accuracy
-      value: 59.252835223482336
+      value: 53.32555036691128
     - type: f1
-      value: 42.617395241215604
+      value: 37.93993994854238
   - task:
       type: Classification
     dataset:
@@ -53,9 +53,9 @@ model-index:
       revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
     metrics:
     - type: accuracy
-      value: 63.03967720242098
+      value: 60.178211163416286
     - type: f1
-      value: 62.32575868187973
+      value: 60.03972849482204
   - task:
       type: Classification
     dataset:
@@ -66,9 +66,9 @@ model-index:
       revision: 7d571f92784cd94a019292a1f45445077d0ef634
     metrics:
     - type: accuracy
-      value: 67.80094149293879
+      value: 64.1930060524546
     - type: f1
-      value: 67.470213025305
+      value: 64.02551940330545
   - task:
       type: STS
     dataset:
@@ -79,17 +79,17 @@ model-index:
       revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
     metrics:
     - type: cos_sim_pearson
-      value: 88.12981800561491
+      value: 88.27146953394649
     - type: cos_sim_spearman
-      value: 87.76573047550748
+      value: 88.57657349986717
     - type: euclidean_pearson
-      value: 88.20566152098061
+      value: 89.07302053356283
     - type: euclidean_spearman
-      value: 87.76635475212959
+      value: 88.57431075359631
     - type: manhattan_pearson
-      value: 88.25680923613729
+      value: 89.11306099359507
     - type: manhattan_spearman
-      value: 87.91724088147261
+      value: 88.61591595032135
   - task:
       type: STS
     dataset:
@@ -100,15 +100,15 @@ model-index:
       revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
     metrics:
     - type: cos_sim_pearson
-      value: 66.96543486888109
+      value: 65.57570700756301
     - type: cos_sim_spearman
-      value: 70.98778280592913
+      value: 70.33230514473709
     - type: euclidean_pearson
-      value: 68.49981525066747
+      value: 66.56411956377407
     - type: euclidean_spearman
-      value: 70.98778280592913
+      value: 70.33230514473709
     - type: manhattan_pearson
-      value: 68.4064756070659
+      value: 66.32203605146594
     - type: manhattan_spearman
-      value: 71.04079668196594
+      value: 70.34809182527435
 ---

app.py

@@ -37,7 +37,7 @@ def retrieval_dataframe_update():
 block = gr.Blocks()
 with block:
     gr.Markdown(f"""**Leaderboard de modelos de Embeddings en español
-    Massive Text Embedding Benchmark (MTEB) Leaderboard.**
+    Massive Spanish Text Embedding Benchmark (MSTEB) Leaderboard.**
     - **Total Datasets**: {NUM_DATASETS}
     - **Total Languages**: 1
     - **Total Scores**: {NUM_SCORES}

data/classification.csv

@@ -1,19 +1,25 @@
 Model name,Average,MTEB AmazonReviewsClassification (es),MTEB MTOPDomainClassification (es),MTEB MTOPIntentClassification (es),MTEB MassiveIntentClassification (es),MTEB MassiveScenarioClassification (es)
-multilingual-e5-large,66.59,42.66,89.95,66.84,64.68,68.85
-bge-small-en-v1.5,52.86,32.03,76.93,52.15,48.77,54.42
-multilingual-e5-base,63.87,42.47,89.62,60.27,60.51,66.52
-multilingual-e5-small,61.13,41.3,87.33,55.87,58.06,63.1
-paraphrase-multilingual-mpnet-base-v2,65.67,39.99,86.96,66.59,64.43,70.42
-sentence-t5-large,57.87,42.89,80.78,52.07,54.1,59.56
+multilingual-e5-LARGE,66.59,42.66,89.95,66.84,64.68,68.85
+bge-SMALL-en-v1.5,52.86,32.03,76.93,52.15,48.77,54.42
+multilingual-e5-BASE,63.87,42.47,89.62,60.27,60.51,66.52
+multilingual-e5-SMALL,61.13,41.3,87.33,55.87,58.06,63.1
+paraphrase-multilingual-mpnet-BASE-v2,65.67,39.99,86.96,66.59,64.43,70.42
+sentence-t5-LARGE,57.87,42.89,80.78,52.07,54.1,59.56
 sentence-t5-xl,61.64,45.01,85.32,57.38,57.97,62.52
 paraphrase-spanish-distilroberta,63.98,38.24,86.81,65.94,60.52,68.39
 sentence_similarity_spanish_es,61.77,35.08,85.86,65.21,58.51,64.21
-paraphrase-multilingual-mpnet-base-v2-ft-stsb_multi_mt-embeddings,64.0,37.25,86.93,66.28,62.6,66.96
-mstsb-paraphrase-multilingual-mpnet-base-v2,64.47,38.29,86.04,67.06,63.47,67.53
-multilingual-e5-base-b16-e10,65.09,43.4,89.02,61.7,63.06,68.25
-multilingual-e5-large-stsb-tuned-b32-e10,66.19,43.31,89.3,64.04,64.62,69.69
-multilingual-e5-large-stsb-tuned-b16-e10,67.1,43.72,90.29,65.51,65.13,70.84
-multilingual-e5-large-stsb-tuned,66.23,43.62,89.33,62.93,65.11,70.16
-multilingual-e5-large-stsb-tuned-b64-e10,64.58,43.71,88.84,60.2,62.74,67.4
+paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,64.0,37.25,86.93,66.28,62.6,66.96
+mstsb-paraphrase-multilingual-mpnet-BASE-v2,64.47,38.29,86.04,67.06,63.47,67.53
+multilingual-e5-BASE-b16-e10,65.09,43.4,89.02,61.7,63.06,68.25
+multilingual-e5-LARGE-stsb-tuned-b32-e10,66.19,43.31,89.3,64.04,64.62,69.69
+multilingual-e5-LARGE-stsb-tuned-b16-e10,67.1,43.72,90.29,65.51,65.13,70.84
+multilingual-e5-LARGE-stsb-tuned-b16-e5,66.23,43.62,89.33,62.93,65.11,70.16
+multilingual-e5-LARGE-stsb-tuned-b64-e10,64.58,43.71,88.84,60.2,62.74,67.4
 LaBSE,61.97,39.39,84.07,64.44,58.32,63.61
-multilingual-e5-base-stsb-tuned-b16-e10-all-languages,64.39,43.04,88.81,59.25,63.04,67.8
+multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,64.39,43.04,88.81,59.25,63.04,67.8
+bge-BASE-tuned-b16-e10,50.83,31.34,74.1,45.63,48.72,54.36
+multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,64.93,42.98,87.96,60.91,63.78,69.02
+bge-LARGE-tuned-b16-e10,51.67,31.72,76.04,48.01,48.9,53.66
+multilingual-e5-LARGE-stsb-tuned-b16-e15,67.56,43.35,90.98,67.27,65.35,70.87
+multilingual-e5-LARGE-stsb-tuned-b64-e10-all-languages,64.85,43.91,87.2,62.25,63.09,67.83
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,63.11,44.38,85.51,57.27,62.15,66.25

data/general.csv

@@ -1,19 +1,25 @@
 Model name,Model Size (GB),Embedding Dimensions,Average,Classification Average,Clustering Average,STS Average,Retrieval Average
-multilingual-e5-large,,,72.22,66.6,,77.83,
-bge-small-en-v1.5,,,59.73,52.86,,66.6,
-multilingual-e5-base,,,70.7,63.88,,77.53,
-multilingual-e5-small,,,68.64,61.13,,76.15,
-paraphrase-multilingual-mpnet-base-v2,,,69.1,65.68,,72.53,
-sentence-t5-large,,,64.04,57.88,,70.21,
+multilingual-e5-LARGE,,,72.22,66.6,,77.83,
+bge-SMALL-en-v1.5,,,59.73,52.86,,66.6,
+multilingual-e5-BASE,,,70.7,63.88,,77.53,
+multilingual-e5-SMALL,,,68.64,61.13,,76.15,
+paraphrase-multilingual-mpnet-BASE-v2,,,69.1,65.68,,72.53,
+sentence-t5-LARGE,,,64.04,57.88,,70.21,
 sentence-t5-xl,,,66.22,61.64,,70.79,
 paraphrase-spanish-distilroberta,,,69.34,63.98,,74.7,
 sentence_similarity_spanish_es,,,68.5,61.77,,75.22,
-paraphrase-multilingual-mpnet-base-v2-ft-stsb_multi_mt-embeddings,,,68.62,64.0,,73.25,
-mstsb-paraphrase-multilingual-mpnet-base-v2,,,69.39,64.48,,74.29,
-multilingual-e5-base-b16-e10,,,71.97,65.09,,78.86,
-multilingual-e5-large-stsb-tuned-b32-e10,,,72.73,66.19,,79.27,
-multilingual-e5-large-stsb-tuned-b16-e10,,,73.07,67.1,,79.05,
-multilingual-e5-large-stsb-tuned,,,72.84,66.23,,79.46,
-multilingual-e5-large-stsb-tuned-b64-e10,,,71.83,64.58,,79.08,
+paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,,,68.62,64.0,,73.25,
+mstsb-paraphrase-multilingual-mpnet-BASE-v2,,,69.39,64.48,,74.29,
+multilingual-e5-BASE-b16-e10,,,71.97,65.09,,78.86,
+multilingual-e5-LARGE-stsb-tuned-b32-e10,,,72.73,66.19,,79.27,
+multilingual-e5-LARGE-stsb-tuned-b16-e10,,,73.07,67.1,,79.05,
+multilingual-e5-LARGE-stsb-tuned-b16-e5,,,72.84,66.23,,79.46,
+multilingual-e5-LARGE-stsb-tuned-b64-e10,,,71.83,64.58,,79.08,
 LaBSE,,,66.99,61.97,,72.01,
-multilingual-e5-base-stsb-tuned-b16-e10-all-languages,,,71.88,64.39,,79.38,
+multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,,,71.88,64.39,,79.38,
+bge-BASE-tuned-b16-e10,,,59.69,50.83,,68.55,
+multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,,,72.22,64.93,,79.5,
+bge-LARGE-tuned-b16-e10,,,61.5,51.67,,71.34,
+multilingual-e5-LARGE-stsb-tuned-b16-e15,,,73.38,67.56,,79.19,
+multilingual-e5-LARGE-stsb-tuned-b64-e10-all-languages,,,71.68,64.85,,78.52,
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,,,71.28,63.11,,79.44,

data/sts.csv

@@ -1,19 +1,25 @@
 Model name,Average,MTEB STS17 (es-es),MTEB STS22 (es)
-multilingual-e5-large,77.82,87.42,68.23
-bge-small-en-v1.5,66.6,77.73,55.47
-multilingual-e5-base,77.52,87.26,67.79
-multilingual-e5-small,76.15,85.27,67.04
-paraphrase-multilingual-mpnet-base-v2,72.52,85.14,59.91
-sentence-t5-large,70.21,82.74,57.68
+multilingual-e5-LARGE,77.82,87.42,68.23
+bge-SMALL-en-v1.5,66.6,77.73,55.47
+multilingual-e5-BASE,77.52,87.26,67.79
+multilingual-e5-SMALL,76.15,85.27,67.04
+paraphrase-multilingual-mpnet-BASE-v2,72.52,85.14,59.91
+sentence-t5-LARGE,70.21,82.74,57.68
 sentence-t5-xl,70.78,83.42,58.16
 paraphrase-spanish-distilroberta,74.7,85.79,63.61
 sentence_similarity_spanish_es,75.22,85.37,65.07
-paraphrase-multilingual-mpnet-base-v2-ft-stsb_multi_mt-embeddings,73.24,86.89,59.6
-mstsb-paraphrase-multilingual-mpnet-base-v2,74.28,88.22,60.36
-multilingual-e5-base-b16-e10,78.86,87.51,70.21
-multilingual-e5-large-stsb-tuned-b32-e10,79.27,88.1,70.44
-multilingual-e5-large-stsb-tuned-b16-e10,79.05,88.53,69.58
-multilingual-e5-large-stsb-tuned,79.46,88.44,70.48
-multilingual-e5-large-stsb-tuned-b64-e10,79.08,88.03,70.12
+paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,73.24,86.89,59.6
+mstsb-paraphrase-multilingual-mpnet-BASE-v2,74.28,88.22,60.36
+multilingual-e5-BASE-b16-e10,78.86,87.51,70.21
+multilingual-e5-LARGE-stsb-tuned-b32-e10,79.27,88.1,70.44
+multilingual-e5-LARGE-stsb-tuned-b16-e10,79.05,88.53,69.58
+multilingual-e5-LARGE-stsb-tuned-b16-e5,79.46,88.44,70.48
+multilingual-e5-LARGE-stsb-tuned-b64-e10,79.08,88.03,70.12
 LaBSE,72.01,80.83,63.18
-multilingual-e5-base-stsb-tuned-b16-e10-all-languages,79.38,87.77,70.99
+multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,79.38,87.77,70.99
+bge-BASE-tuned-b16-e10,68.55,82.01,55.08
+multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,79.5,88.66,70.35
+bge-LARGE-tuned-b16-e10,71.34,81.57,61.12
+multilingual-e5-LARGE-stsb-tuned-b16-e15,79.19,89.04,69.34
+multilingual-e5-LARGE-stsb-tuned-b64-e10-all-languages,78.52,88.37,68.67
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,79.44,88.46,70.41