feat: multilingual tab and data

main.py

@@ -53,6 +53,55 @@ df_mah_pivot.sort_values(by="Average ⬆️", ascending=False, inplace=True)
 df_mah_pivot.index.rename("Model", inplace=True)
 df_mah_pivot.reset_index(drop=False, inplace=True)
 
+df_eval = pd.read_csv("multilingual_results.csv")
+
+def map_task_to_group(task: str) -> str | None:
+    if task == "xcopa":
+        return "XCOPA"
+    if task == "xstorycloze":
+        return "XStoryCloze"
+    if task == "xwinograd":
+        return "XWinograd"
+    if task.startswith("include_base_44_"):
+        return "INCLUDE"
+    if task.startswith("belebele_"):
+        return "Belebele"
+    if task.startswith("global_mmlu_full_"):
+        return "Global MMLU"
+    return None
+
+df_eval["group"] = df_eval.task.apply(map_task_to_group)
+df_eval_grouped = df_eval[df_eval["group"].notna()].copy()
+df_eval_grouped["Model"] = df_eval_grouped.model_name.apply(lambda s: s.split("/")[-1])
+df_multilingual_pivot = df_eval_grouped.pivot_table(
+    index="Model", columns="group", values="performance", aggfunc="mean"
+)
+df_multilingual_pivot["Average ⬆️"] = df_multilingual_pivot.mean(axis=1)
+df_multilingual_pivot.sort_values(by="Average ⬆️", ascending=False, inplace=True)
+df_multilingual_pivot.index.rename("Model", inplace=True)
+df_multilingual_pivot.reset_index(drop=False, inplace=True)
+
+# Determine display names for groups including n_shot when unique
+group_nshot = (
+    df_eval_grouped.groupby("group")["n_shot"]
+    .agg(lambda s: s.iloc[0] if s.nunique() == 1 else "mixed")
+    .to_dict()
+)
+
+def display_name(group: str) -> str:
+    label = group_nshot.get(group, "unknown")
+    if label == "mixed" or label == "unknown" or label == "unknown":
+        return f"{group} [mixed]" if label == "mixed" else f"{group} [unknown]"
+    return f"{group} [{label}]"
+
+# Build a renamed version for display, preserving Model and Average columns
+display_columns_map = {
+    col: display_name(col)
+    for col in df_multilingual_pivot.columns
+    if col not in ["Model", "Average ⬆️"]
+}
+df_multilingual_display_all = df_multilingual_pivot.rename(columns=display_columns_map)
+
 cols = [
     #'Llama-3.1-8B',
     "Llama-3.1-Tulu-3-8B-SFT",
@@ -146,6 +195,42 @@ with gr.Blocks() as demo:
                 ),
             )
 
+        with gr.Tab("Multilingual evaluations 🌍"):
+            gr.Markdown(
+                """
+            Aggregated multilingual performance by task group (mean across languages when applicable).
+            """
+            )
+            # Order columns: Model, groups..., Average
+            raw_group_columns = [
+                col
+                for col in [
+                    "INCLUDE",
+                    "Belebele",
+                    "Global MMLU",
+                    "XCOPA",
+                    "XStoryCloze",
+                    "XWinograd",
+                ]
+                if col in df_multilingual_pivot.columns
+            ]
+            display_group_columns = [display_columns_map[col] for col in raw_group_columns]
+            ordered_columns = ["Model", *display_group_columns, "Average ⬆️"]
+            df_multilingual_display = df_multilingual_display_all.loc[:, ordered_columns]
+            Leaderboard(
+                value=df_multilingual_display.round(2),
+                select_columns=SelectColumns(
+                    default_selection=list(df_multilingual_display.columns),
+                    cant_deselect=["Model"],
+                    label="Select Columns to Display:",
+                ),
+                search_columns=SearchColumns(
+                    primary_column="Model",
+                    label="Filter a model",
+                    secondary_columns=[],
+                ),
+            )
+
 
 if __name__ == "__main__":
     demo.launch()

multilingual_results.csv

@@ -0,0 +1,338 @@
+model_name,task,n_shot,performance
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_italian,0,0.2937956204379562
+HuggingFaceTB/SmolLM3-3B,belebele_swe_Latn,5,0.69
+HuggingFaceTB/SmolLM3-3B,include_base_44_hungarian,0,0.2818181818181818
+Qwen/Qwen3-1.7B,belebele_est_Latn,5,0.56
+HuggingFaceTB/SmolLM2-1.7B,belebele_dan_Latn,5,0.3466666666666667
+HuggingFaceTB/SmolLM3-3B,include_base_44_georgian,0,0.276
+HuggingFaceTB/SmolLM3-3B,belebele_por_Latn,5,0.7944444444444444
+google/gemma-3-4b-it,global_mmlu_full_pt,5,0.527204101979775
+HuggingFaceTB/SmolLM3-3B,include_base_44_estonian,0,0.29464285714285715
+google/gemma-3-4b-it,belebele_slv_Latn,5,0.7533333333333333
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_pl,5,0.31825950719270757
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_it,5,0.35878080045577554
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_bulgarian,0,0.28909090909090907
+HuggingFaceTB/SmolLM3-3B,belebele_hrv_Latn,5,0.5722222222222222
+google/gemma-3-4b-it,include_base_44_serbian,0,0.5672727272727273
+HuggingFaceTB/SmolLM3-3B,belebele_mlt_Latn,5,0.3566666666666667
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_spanish,0,0.3181818181818182
+Qwen/Qwen3-1.7B,belebele_fra_Latn,5,0.7922222222222223
+google/gemma-3-4b-it,include_base_44_estonian,0,0.47767857142857145
+google/gemma-3-4b-it,include_base_44_dutch,0,0.5517241379310345
+HuggingFaceTB/SmolLM3-3B,include_base_44_dutch,0,0.47005444646098005
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_es,5,0.3245976356644353
+Qwen/Qwen3-1.7B,include_base_44_lithuanian,0,0.37265917602996257
+Qwen/Qwen3-1.7B,include_base_44_polish,0,0.4142335766423358
+Qwen/Qwen3-1.7B,global_mmlu_full_cs,5,0.4690927218344965
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_deu_Latn,5,0.33444444444444443
+HuggingFaceTB/SmolLM2-1.7B,belebele_ita_Latn,5,0.4033333333333333
+HuggingFaceTB/SmolLM3-3B,belebele_ell_Grek,5,0.7633333333333333
+HuggingFaceTB/SmolLM3-3B,belebele_dan_Latn,5,0.6522222222222223
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_lithuanian,0,0.27153558052434457
+Qwen/Qwen3-1.7B,belebele_eng_Latn,5,0.8255555555555556
+Qwen/Qwen3-1.7B,belebele_hun_Latn,5,0.6733333333333333
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_ro,5,0.451502634952286
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_serbian,0,0.28363636363636363
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_belarusian,0,0.26
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_german,0,0.35251798561151076
+HuggingFaceTB/SmolLM2-1.7B,belebele_nld_Latn,5,0.41333333333333333
+google/gemma-3-4b-it,global_mmlu_full_sr,5,0.4661016949152542
+google/gemma-3-4b-it,global_mmlu_full_ro,5,0.5182310212220481
+google/gemma-3-4b-it,include_base_44_north macedonian,0,0.6678765880217786
+google/gemma-3-4b-it,include_base_44_hungarian,0,0.41454545454545455
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_hrv_Latn,5,0.27
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_serbian,0,0.22545454545454546
+HuggingFaceTB/SmolLM2-1.7B,belebele_hrv_Latn,5,0.33666666666666667
+Qwen/Qwen3-1.7B,belebele_ita_Latn,5,0.7511111111111111
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_basque,0,0.278
+google/gemma-3-4b-it,include_base_44_portuguese,0,0.49364791288566245
+Qwen/Qwen3-1.7B,include_base_44_hungarian,0,0.37636363636363634
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_pt,5,0.3721692066657171
+google/gemma-3-4b-it,global_mmlu_full_tr,5,0.4814129041447087
+google/gemma-3-4b-it,xcopa,0,0.6247272727272727
+HuggingFaceTB/SmolLM3-3B,include_base_44_greek,0,0.41304347826086957
+Qwen/Qwen3-1.7B,include_base_44_basque,0,0.316
+HuggingFaceTB/SmolLM3-3B,belebele_ces_Latn,5,0.6222222222222222
+Qwen/Qwen3-1.7B,global_mmlu_full_sv,5,0.4794188861985472
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_pol_Latn,5,0.28555555555555556
+HuggingFaceTB/SmolLM2-1.7B,belebele_pol_Latn,5,0.3288888888888889
+Qwen/Qwen3-1.7B,belebele_deu_Latn,5,0.74
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_ru,5,0.3166215638797892
+HuggingFaceTB/SmolLM2-1.7B,belebele_eng_Latn,5,0.5633333333333334
+Qwen/Qwen3-1.7B,belebele_lit_Latn,5,0.6266666666666667
+HuggingFaceTB/SmolLM2-1.7B,belebele_deu_Latn,5,0.3933333333333333
+google/gemma-3-4b-it,belebele_ces_Latn,5,0.7733333333333333
+google/gemma-3-4b-it,include_base_44_croatian,0,0.6236363636363637
+HuggingFaceTB/SmolLM2-1.7B,belebele_est_Latn,5,0.2922222222222222
+HuggingFaceTB/SmolLM3-3B,belebele_slk_Latn,5,0.5855555555555556
+HuggingFaceTB/SmolLM3-3B,include_base_44_north macedonian,0,0.5081669691470054
+HuggingFaceTB/SmolLM3-3B,include_base_44_armenian,0,0.2581818181818182
+HuggingFaceTB/SmolLM2-1.7B,belebele_fin_Latn,5,0.32
+HuggingFaceTB/SmolLM3-3B,belebele_deu_Latn,5,0.8155555555555556
+Qwen/Qwen3-1.7B,belebele_ell_Grek,5,0.6688888888888889
+HuggingFaceTB/SmolLM2-1.7B,belebele_slv_Latn,5,0.3233333333333333
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_ro,5,0.28051559606893606
+google/gemma-3-4b-it,global_mmlu_full_de,5,0.5190856003418316
+Qwen/Qwen3-1.7B,include_base_44_north macedonian,0,0.5353901996370236
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_fr,5,0.32068081469876086
+Qwen/Qwen3-1.7B,xcopa,0,0.5750909090909091
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_bul_Cyrl,5,0.3211111111111111
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_lt,5,0.29155390969947304
+HuggingFaceTB/SmolLM2-1.7B,belebele_lvs_Latn,5,0.33
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_el,5,0.2901296111665005
+google/gemma-3-4b-it,include_base_44_albanian,0,0.5753176043557169
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_dutch,0,0.33030852994555354
+HuggingFaceTB/SmolLM2-1.7B,belebele_lit_Latn,5,0.32
+HuggingFaceTB/SmolLM2-1.7B,belebele_bul_Cyrl,5,0.3244444444444444
+HuggingFaceTB/SmolLM3-3B,include_base_44_belarusian,0,0.24909090909090909
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_uk,5,0.4272183449651047
+google/gemma-3-4b-it,include_base_44_bulgarian,0,0.6127272727272727
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_azerbaijani,0,0.26094890510948904
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_croatian,0,0.25272727272727274
+HuggingFaceTB/SmolLM2-1.7B,belebele_spa_Latn,5,0.45666666666666667
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_russian,0,0.28442028985507245
+HuggingFaceTB/SmolLM3-3B,include_base_44_albanian,0,0.3666061705989111
+Qwen/Qwen3-1.7B,global_mmlu_full_tr,5,0.44281441390115367
+HuggingFaceTB/SmolLM3-3B,belebele_nld_Latn,5,0.6777777777777778
+Qwen/Qwen3-1.7B,include_base_44_azerbaijani,0,0.39233576642335766
+google/gemma-3-4b-it,belebele_fin_Latn,5,0.7744444444444445
+Qwen/Qwen3-1.7B,global_mmlu_full_sr,5,0.43355647343683235
+Qwen/Qwen3-1.7B,global_mmlu_full_en,5,0.6010539809143997
+google/gemma-3-4b-it,belebele_hrv_Latn,5,0.7711111111111111
+google/gemma-3-4b-it,belebele_bul_Cyrl,5,0.7744444444444445
+HuggingFaceTB/SmolLM3-3B,include_base_44_basque,0,0.318
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_greek,0,0.25181159420289856
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_el,5,0.480985614584817
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_polish,0,0.25364963503649635
+HuggingFaceTB/SmolLM3-3B,include_base_44_polish,0,0.34124087591240876
+Qwen/Qwen3-1.7B,global_mmlu_full_ru,5,0.5029198119925936
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_tr,5,0.3821392963965247
+Qwen/Qwen3-1.7B,include_base_44_ukrainian,0,0.49272727272727274
+HuggingFaceTB/SmolLM3-3B,include_base_44_lithuanian,0,0.3389513108614232
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_slv_Latn,5,0.27666666666666667
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_ukrainian,0,0.36
+Qwen/Qwen3-1.7B,include_base_44_georgian,0,0.314
+Qwen/Qwen3-1.7B,include_base_44_estonian,0,0.3705357142857143
+Qwen/Qwen3-1.7B,belebele_nld_Latn,5,0.7166666666666667
+HuggingFaceTB/SmolLM2-1.7B,belebele_ces_Latn,5,0.33
+google/gemma-3-4b-it,belebele_mlt_Latn,5,0.65
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_uk,5,0.2947585813986612
+HuggingFaceTB/SmolLM3-3B,belebele_bul_Cyrl,5,0.6366666666666667
+Qwen/Qwen3-1.7B,belebele_bul_Cyrl,5,0.6966666666666667
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_he,5,0.24896738356359494
+HuggingFaceTB/SmolLM3-3B,xstorycloze,0,0.6174718729318333
+HuggingFaceTB/SmolLM3-3B,xcopa,0,0.5889090909090909
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_turkish,0,0.23905109489051096
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_dan_Latn,5,0.30666666666666664
+Qwen/Qwen3-1.7B,include_base_44_spanish,0,0.5472727272727272
+HuggingFaceTB/SmolLM2-1.7B,belebele_mlt_Latn,5,0.3
+google/gemma-3-4b-it,include_base_44_azerbaijani,0,0.40693430656934304
+google/gemma-3-4b-it,belebele_dan_Latn,5,0.7888888888888889
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_mlt_Latn,5,0.32222222222222224
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,xwinograd,0,0.6983591818386155
+google/gemma-3-4b-it,include_base_44_russian,0,0.4782608695652174
+google/gemma-3-4b-it,global_mmlu_full_cs,5,0.5044865403788634
+google/gemma-3-4b-it,global_mmlu_full_it,5,0.5321179319185301
+Qwen/Qwen3-1.7B,belebele_por_Latn,5,0.7644444444444445
+Qwen/Qwen3-1.7B,include_base_44_greek,0,0.3713768115942029
+google/gemma-3-4b-it,include_base_44_french,0,0.548926014319809
+google/gemma-3-4b-it,belebele_lit_Latn,5,0.7366666666666667
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_finnish,0,0.24682395644283123
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_fr,5,0.5495655889474433
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_ron_Latn,5,0.31222222222222223
+Qwen/Qwen3-1.7B,global_mmlu_full_pt,5,0.5347528842045293
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_it,5,0.30366044722973934
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_azerbaijani,0,0.2791970802919708
+google/gemma-3-4b-it,belebele_spa_Latn,5,0.7722222222222223
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_russian,0,0.26992753623188404
+Qwen/Qwen3-1.7B,global_mmlu_full_nl,5,0.4950149551345962
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_hungarian,0,0.24363636363636362
+Qwen/Qwen3-1.7B,belebele_swe_Latn,5,0.73
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_georgian,0,0.252
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_turkish,0,0.2208029197080292
+google/gemma-3-4b-it,global_mmlu_full_sv,5,0.5211508332146418
+HuggingFaceTB/SmolLM3-3B,belebele_ita_Latn,5,0.7711111111111111
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_north macedonian,0,0.2631578947368421
+Qwen/Qwen3-1.7B,belebele_mlt_Latn,5,0.43444444444444447
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_nld_Latn,5,0.3288888888888889
+Qwen/Qwen3-1.7B,belebele_pol_Latn,5,0.7244444444444444
+HuggingFaceTB/SmolLM2-1.7B,belebele_por_Latn,5,0.4633333333333333
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_pl,5,0.25986326734083465
+Qwen/Qwen3-1.7B,belebele_fin_Latn,5,0.6155555555555555
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_nl,5,0.4615439396097422
+HuggingFaceTB/SmolLM3-3B,belebele_slv_Latn,5,0.49666666666666665
+google/gemma-3-4b-it,include_base_44_belarusian,0,0.2872727272727273
+HuggingFaceTB/SmolLM2-1.7B,belebele_slk_Latn,5,0.32222222222222224
+Qwen/Qwen3-1.7B,include_base_44_armenian,0,0.3054545454545455
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_swe_Latn,5,0.33444444444444443
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_es,5,0.551345962113659
+HuggingFaceTB/SmolLM3-3B,include_base_44_russian,0,0.4492753623188406
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_nl,5,0.3525851018373451
+google/gemma-3-4b-it,include_base_44_turkish,0,0.5237226277372263
+google/gemma-3-4b-it,belebele_eng_Latn,5,0.8555555555555555
+google/gemma-3-4b-it,belebele_swe_Latn,5,0.7955555555555556
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_cs,5,0.31932773109243695
+HuggingFaceTB/SmolLM3-3B,include_base_44_turkish,0,0.3795620437956204
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_greek,0,0.2554347826086957
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_albanian,0,0.2613430127041742
+Qwen/Qwen3-1.7B,include_base_44_portuguese,0,0.4791288566243194
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_lt,5,0.3356359492949722
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_slk_Latn,5,0.3233333333333333
+HuggingFaceTB/SmolLM3-3B,include_base_44_portuguese,0,0.5045372050816697
+Qwen/Qwen3-1.7B,include_base_44_french,0,0.48448687350835323
+Qwen/Qwen3-1.7B,belebele_slk_Latn,5,0.6788888888888889
+google/gemma-3-4b-it,include_base_44_spanish,0,0.5690909090909091
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_nl,5,0.2975359635379576
+Qwen/Qwen3-1.7B,global_mmlu_full_it,5,0.5266343825665859
+google/gemma-3-4b-it,include_base_44_basque,0,0.356
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_de,5,0.5394530693633386
+Qwen/Qwen3-1.7B,belebele_ces_Latn,5,0.71
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_lit_Latn,5,0.30333333333333334
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_french,0,0.3317422434367542
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_georgian,0,0.258
+google/gemma-3-4b-it,include_base_44_ukrainian,0,0.5836363636363636
+HuggingFaceTB/SmolLM3-3B,belebele_hun_Latn,5,0.48
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_german,0,0.3381294964028777
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_sv,5,0.2956843754450933
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_fr,5,0.3785785500640934
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_belarusian,0,0.2672727272727273
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,xstorycloze,0,0.515071295349257
+HuggingFaceTB/SmolLM3-3B,include_base_44_ukrainian,0,0.46545454545454545
+HuggingFaceTB/SmolLM3-3B,belebele_spa_Latn,5,0.7955555555555556
+HuggingFaceTB/SmolLM3-3B,include_base_44_german,0,0.43884892086330934
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_north macedonian,0,0.2395644283121597
+google/gemma-3-4b-it,belebele_slk_Latn,5,0.7811111111111111
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_finnish,0,0.279491833030853
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_pl,5,0.4150405925081897
+google/gemma-3-4b-it,include_base_44_polish,0,0.4635036496350365
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,xcopa,0,0.518
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_bulgarian,0,0.27090909090909093
+Qwen/Qwen3-1.7B,include_base_44_russian,0,0.4746376811594203
+Qwen/Qwen3-1.7B,belebele_ron_Latn,5,0.72
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_lt,5,0.2622133599202393
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_es,5,0.38199686654322745
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_pt,5,0.543156245549067
+google/gemma-3-4b-it,belebele_nld_Latn,5,0.7766666666666666
+google/gemma-3-4b-it,belebele_ita_Latn,5,0.7866666666666666
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_cs,5,0.2676969092721834
+Qwen/Qwen3-1.7B,include_base_44_bulgarian,0,0.48
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_spa_Latn,5,0.37444444444444447
+google/gemma-3-4b-it,global_mmlu_full_ru,5,0.5084745762711864
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_he,5,0.28165503489531407
+google/gemma-3-4b-it,global_mmlu_full_en,5,0.5827517447657029
+google/gemma-3-4b-it,global_mmlu_full_fr,5,0.5274889616863695
+HuggingFaceTB/SmolLM2-1.7B,belebele_hun_Latn,5,0.3011111111111111
+google/gemma-3-4b-it,belebele_ell_Grek,5,0.7922222222222223
+Qwen/Qwen3-1.7B,include_base_44_german,0,0.45323741007194246
+google/gemma-3-4b-it,global_mmlu_full_el,5,0.48511608033043724
+google/gemma-3-4b-it,global_mmlu_full_pl,5,0.5032046716991881
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_cs,5,0.4178179746474861
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_ces_Latn,5,0.30333333333333334
+Qwen/Qwen3-1.7B,xstorycloze,0,0.5671740569159497
+Qwen/Qwen3-1.7B,include_base_44_finnish,0,0.3720508166969147
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_croatian,0,0.27454545454545454
+google/gemma-3-4b-it,belebele_deu_Latn,5,0.8066666666666666
+Qwen/Qwen3-1.7B,global_mmlu_full_uk,5,0.45748468879077053
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_polish,0,0.26094890510948904
+Qwen/Qwen3-1.7B,xwinograd,0,0.7145425938413127
+google/gemma-3-4b-it,xwinograd,0,0.7772533153517645
+Qwen/Qwen3-1.7B,include_base_44_dutch,0,0.484573502722323
+Qwen/Qwen3-1.7B,include_base_44_serbian,0,0.44363636363636366
+Qwen/Qwen3-1.7B,include_base_44_belarusian,0,0.3
+Qwen/Qwen3-1.7B,global_mmlu_full_he,5,0.4016521862982481
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_estonian,0,0.24553571428571427
+HuggingFaceTB/SmolLM3-3B,include_base_44_italian,0,0.6441605839416058
+Qwen/Qwen3-1.7B,global_mmlu_full_es,5,0.5344680244979347
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_sr,5,0.3767981768978778
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_sr,5,0.25523429710867396
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_en,5,0.498860561173622
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_fra_Latn,5,0.3488888888888889
+HuggingFaceTB/SmolLM3-3B,belebele_fin_Latn,5,0.46555555555555556
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_eng_Latn,5,0.5866666666666667
+HuggingFaceTB/SmolLM3-3B,include_base_44_finnish,0,0.3393829401088929
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_he,5,0.3240991311778949
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_fin_Latn,5,0.3
+HuggingFaceTB/SmolLM3-3B,belebele_ron_Latn,5,0.6511111111111111
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_portuguese,0,0.30671506352087113
+google/gemma-3-4b-it,xstorycloze,0,0.668732326574815
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_hun_Latn,5,0.23666666666666666
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_por_Latn,5,0.36
+HuggingFaceTB/SmolLM3-3B,belebele_fra_Latn,5,0.8055555555555556
+google/gemma-3-4b-it,include_base_44_georgian,0,0.526
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_spanish,0,0.3327272727272727
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_tr,5,0.31035465033471016
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_dutch,0,0.2250453720508167
+google/gemma-3-4b-it,belebele_hun_Latn,5,0.7322222222222222
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_french,0,0.3054892601431981
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_ell_Grek,5,0.28444444444444444
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_sv,5,0.436476285429426
+HuggingFaceTB/SmolLM2-1.7B,belebele_ell_Grek,5,0.31444444444444447
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_hungarian,0,0.27090909090909093
+Qwen/Qwen3-1.7B,belebele_spa_Latn,5,0.7844444444444445
+Qwen/Qwen3-1.7B,include_base_44_albanian,0,0.455535390199637
+HuggingFaceTB/SmolLM3-3B,belebele_eng_Latn,5,0.8488888888888889
+google/gemma-3-4b-it,belebele_est_Latn,5,0.6988888888888889
+HuggingFaceTB/SmolLM3-3B,include_base_44_azerbaijani,0,0.31204379562043794
+google/gemma-3-4b-it,belebele_fra_Latn,5,0.82
+google/gemma-3-4b-it,include_base_44_armenian,0,0.3472727272727273
+google/gemma-3-4b-it,belebele_por_Latn,5,0.7866666666666666
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_lithuanian,0,0.26779026217228463
+Qwen/Qwen3-1.7B,belebele_hrv_Latn,5,0.6966666666666667
+Qwen/Qwen3-1.7B,include_base_44_italian,0,0.5894160583941606
+google/gemma-3-4b-it,belebele_ron_Latn,5,0.78
+HuggingFaceTB/SmolLM3-3B,xwinograd,0,0.7988311980220274
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_ru,5,0.508759435977781
+HuggingFaceTB/SmolLM3-3B,belebele_pol_Latn,5,0.6066666666666667
+HuggingFaceTB/SmolLM3-3B,belebele_lit_Latn,5,0.4388888888888889
+Qwen/Qwen3-1.7B,belebele_lvs_Latn,5,0.6677777777777778
+HuggingFaceTB/SmolLM2-1.7B,xstorycloze,0,0.5408218518741351
+Qwen/Qwen3-1.7B,include_base_44_croatian,0,0.5127272727272727
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_armenian,0,0.2581818181818182
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_ru,5,0.258082894174619
+Qwen/Qwen3-1.7B,include_base_44_turkish,0,0.42700729927007297
+HuggingFaceTB/SmolLM3-3B,include_base_44_croatian,0,0.44
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_tr,5,0.25872382851445663
+Qwen/Qwen3-1.7B,global_mmlu_full_pl,5,0.47393533684660305
+HuggingFaceTB/SmolLM3-3B,belebele_est_Latn,5,0.37
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_est_Latn,5,0.27444444444444444
+google/gemma-3-4b-it,belebele_pol_Latn,5,0.7666666666666667
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_sv,5,0.3335707164221621
+Qwen/Qwen3-1.7B,global_mmlu_full_el,5,0.4171058253809999
+HuggingFaceTB/SmolLM2-1.7B,global_mmlu_full_sr,5,0.2920524141860134
+Qwen/Qwen3-1.7B,global_mmlu_full_de,5,0.5170203674690215
+google/gemma-3-4b-it,global_mmlu_full_he,5,0.4596211365902293
+google/gemma-3-4b-it,include_base_44_greek,0,0.4891304347826087
+google/gemma-3-4b-it,include_base_44_finnish,0,0.4355716878402904
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_estonian,0,0.19196428571428573
+google/gemma-3-4b-it,include_base_44_italian,0,0.6131386861313869
+Qwen/Qwen3-1.7B,belebele_dan_Latn,5,0.7044444444444444
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_el,5,0.2511038313630537
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_en,5,0.48454636091724823
+google/gemma-3-4b-it,global_mmlu_full_uk,5,0.4910269192422732
+Qwen/Qwen3-1.7B,global_mmlu_full_ro,5,0.4859706594502208
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_pt,5,0.3166927788064378
+HuggingFaceTB/SmolLM3-3B,include_base_44_french,0,0.5823389021479713
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_portuguese,0,0.279491833030853
+google/gemma-3-4b-it,global_mmlu_full_nl,5,0.5225751317476143
+HuggingFaceTB/SmolLM3-3B,include_base_44_spanish,0,0.5836363636363636
+HuggingFaceTB/SmolLM3-3B,belebele_lvs_Latn,5,0.41555555555555557
+HuggingFaceTB/SmolLM2-1.7B,xcopa,0,0.5305454545454545
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_en,5,0.6007691212078051
+Qwen/Qwen3-1.7B,global_mmlu_full_fr,5,0.5274177467597209
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_ukrainian,0,0.30727272727272725
+google/gemma-3-4b-it,belebele_lvs_Latn,5,0.7455555555555555
+HuggingFaceTB/SmolLM3-3B,include_base_44_bulgarian,0,0.45636363636363636
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_ita_Latn,5,0.30333333333333334
+HuggingFaceTB/SmolLM2-1.7B,xwinograd,0,0.7289278489548213
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,global_mmlu_full_de,5,0.30957128614157525
+google/gemma-3-4b-it,include_base_44_lithuanian,0,0.5056179775280899
+HuggingFaceTB/SmolLM3-3B,global_mmlu_full_it,5,0.5514883919669563
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,belebele_lvs_Latn,5,0.3011111111111111
+HuggingFaceTB/SmolLM2-1.7B,belebele_ron_Latn,5,0.3711111111111111
+open-sci/open-sci-ref-v0.01-1.7b-nemotron-cc-hq-1T-4096,include_base_44_albanian,0,0.2831215970961887
+Qwen/Qwen3-1.7B,belebele_slv_Latn,5,0.6611111111111111
+google/gemma-3-4b-it,global_mmlu_full_lt,5,0.4684517874946589
+google/gemma-3-4b-it,include_base_44_german,0,0.381294964028777
+Qwen/Qwen3-1.7B,global_mmlu_full_lt,5,0.4099843327161373
+HuggingFaceTB/SmolLM3-3B,include_base_44_serbian,0,0.40545454545454546
+HuggingFaceTB/SmolLM2-1.7B,include_base_44_italian,0,0.29927007299270075