refactor(model_links and slider):

1. Removed the HTML code from core.py
2. Removed the secondary lookup table
3. Now uses the HF database to set up the model meta data.

app.py

@@ -17,9 +17,9 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem(
-            "🏅 LLM accuracy benchmark",
-            elem_id="llm-benchmark-tab-table-acc",
-            id=0,
+                "🏅 LLM accuracy benchmark",
+                elem_id="llm-benchmark-tab-table-acc",
+                id=0,
         ) as acc:
             with gr.Column():
                 with gr.Row():
@@ -44,8 +44,8 @@ with demo:
                             value=list(T_SYMBOLS.values()),
                         )
                     with gr.Column():
-                        model_sizes = RangeSlider(minimum=0,maximum=150,value=(7, 10),label="Select the number of parameters (B)")
-
+                        model_sizes = RangeSlider(minimum=0, maximum=150, value=(7, 10),
+                                                  label="Select the number of parameters (B)")
 
                 with gr.Row():
                     langs_bar = gr.CheckboxGroup(
@@ -95,7 +95,8 @@ with demo:
                     scale=1,
                 )
                 select.click(
-                    lambda: gr.CheckboxGroup(value=core.get_available_task_groups(core.get_selected_task_type(0), True)),
+                    lambda: gr.CheckboxGroup(
+                        value=core.get_available_task_groups(core.get_selected_task_type(0), True)),
                     inputs=[],
                     outputs=shown_tasks,
                 )
@@ -104,9 +105,9 @@ with demo:
             leaderboard_table = gr.Dataframe(datatype=["str", "markdown"], column_widths=[None, "30%"], wrap=False)
 
         with gr.TabItem(
-            "🏅 LLM accuracy benchmark (Zero-Shot)",
-            elem_id="llm-benchmark-tab-table-acc-zeroshot",
-            id=3,
+                "🏅 LLM accuracy benchmark (Zero-Shot)",
+                elem_id="llm-benchmark-tab-table-acc-zeroshot",
+                id=3,
         ) as acc_zero_shot:
             with gr.Column():
                 with gr.Row():
@@ -117,7 +118,6 @@ with demo:
                         elem_id="search-bar",
                     )
 
-
                 with gr.Row():
                     with gr.Column():
                         model_types_zero_shot = gr.CheckboxGroup(
@@ -133,7 +133,7 @@ with demo:
                         )
                     with gr.Column():
                         model_sizes_zero_shot = RangeSlider(minimum=0, maximum=150, value=(7, 10),
-                                                  label="Select the number of parameters (B)")
+                                                            label="Select the number of parameters (B)")
 
                 with gr.Row():
                     langs_bar_zero_shot = gr.CheckboxGroup(
@@ -183,16 +183,18 @@ with demo:
                     scale=1,
                 )
                 select_zero_shot.click(
-                    lambda: gr.CheckboxGroup(value=core.get_available_task_groups(core.get_selected_task_type(3), False)),
+                    lambda: gr.CheckboxGroup(
+                        value=core.get_available_task_groups(core.get_selected_task_type(3), False)),
                     inputs=[],
                     outputs=shown_tasks_zero_shot,
                 )
-            leaderboard_table_zero_shot = gr.Dataframe(datatype=["str", "markdown"], column_widths=[None, "30%"], wrap=False)
+            leaderboard_table_zero_shot = gr.Dataframe(datatype=["str", "markdown"], column_widths=[None, "30%"],
+                                                       wrap=False)
 
         with gr.TabItem(
-            "🌐 LLM translation benchmark",
-            elem_id="llm-benchmark-tab-table-misc",
-            id=1,
+                "🌐 LLM translation benchmark",
+                elem_id="llm-benchmark-tab-table-misc",
+                id=1,
         ) as misc:
             with gr.Column():
                 with gr.Row():
@@ -218,8 +220,7 @@ with demo:
                         )
                     with gr.Column():
                         model_sizes_misc = RangeSlider(minimum=0, maximum=150, value=(7, 10),
-                                                            label="Select the number of parameters (B)")
-
+                                                       label="Select the number of parameters (B)")
 
                 with gr.Row():
                     langs_bar_misc = gr.CheckboxGroup(
@@ -269,7 +270,8 @@ with demo:
                     scale=1,
                 )
                 select_all_tasks_misc.click(
-                    lambda: gr.CheckboxGroup(value=core.get_available_task_groups(core.get_selected_task_type(1), False)),
+                    lambda: gr.CheckboxGroup(
+                        value=core.get_available_task_groups(core.get_selected_task_type(1), False)),
                     inputs=[],
                     outputs=shown_tasks_misc,
                 )
@@ -277,9 +279,9 @@ with demo:
             leaderboard_table_misc = gr.Dataframe(datatype=["str", "markdown"], column_widths=[None, "30%"], wrap=False)
 
         with gr.TabItem(
-            "🌐 LLM MT-Bench benchmark",
-            elem_id="llm-benchmark-tab-table-mtbench",
-            id=2,
+                "🌐 LLM MT-Bench benchmark",
+                elem_id="llm-benchmark-tab-table-mtbench",
+                id=2,
         ) as mtbench:
             with gr.Column():
                 with gr.Row():
@@ -317,7 +319,8 @@ with demo:
                         outputs=langs_bar_mtbench,
                     )
 
-            leaderboard_table_mtbench = gr.Dataframe(datatype=["str", "markdown"], column_widths=[None, "60%"], wrap=False)
+            leaderboard_table_mtbench = gr.Dataframe(datatype=["str", "markdown"], column_widths=[None, "60%"],
+                                                     wrap=False)
 
         for comp, fn in [
             (search_bar, "submit"),
@@ -342,7 +345,8 @@ with demo:
         ]:
             getattr(comp, fn)(
                 core.update_df,
-                [shown_tasks_zero_shot, search_bar_zero_shot, langs_bar_zero_shot, model_types_zero_shot, model_sizes_zero_shot, gr.State(value=False)],
+                [shown_tasks_zero_shot, search_bar_zero_shot, langs_bar_zero_shot, model_types_zero_shot,
+                 model_sizes_zero_shot, gr.State(value=False)],
                 leaderboard_table_zero_shot,
             )
 
@@ -355,7 +359,8 @@ with demo:
         ]:
             getattr(comp, fn)(
                 core.update_df,
-                [shown_tasks_misc, search_bar_misc, langs_bar_misc, model_types_misc, model_sizes_misc, gr.State(value=False)],
+                [shown_tasks_misc, search_bar_misc, langs_bar_misc, model_types_misc, model_sizes_misc,
+                 gr.State(value=False)],
                 leaderboard_table_misc,
             )
 
@@ -365,7 +370,9 @@ with demo:
         ]:
             getattr(comp, fn)(
                 core.update_df,
-                [gr.State(value=core.get_available_task_groups(core.get_selected_task_type(2), False)), search_bar_mtbench, langs_bar_mtbench, gr.State(value=[T_SYMBOLS["chat"]]), gr.State(value=False)],  # TODO
+                [gr.State(value=core.get_available_task_groups(core.get_selected_task_type(2), False)),
+                 search_bar_mtbench, langs_bar_mtbench, gr.State(value=[T_SYMBOLS["chat"]]), gr.State(value=False)],
+                # TODO
                 leaderboard_table_mtbench,
             )
 
@@ -380,21 +387,24 @@ with demo:
     gr.Blocks.load(
         block=demo,
         fn=core.update_df,
-        inputs=[shown_tasks_zero_shot, search_bar_zero_shot, langs_bar_zero_shot, model_types_zero_shot, model_sizes_zero_shot, gr.State(value=False)],
+        inputs=[shown_tasks_zero_shot, search_bar_zero_shot, langs_bar_zero_shot, model_types_zero_shot,
+                model_sizes_zero_shot, gr.State(value=False)],
         outputs=leaderboard_table_zero_shot,
     )
 
     gr.Blocks.load(
         block=demo,
         fn=core.update_df,
-        inputs=[shown_tasks_misc, search_bar_misc, langs_bar_misc, model_types_misc, model_sizes_misc, gr.State(value=False)],
+        inputs=[shown_tasks_misc, search_bar_misc, langs_bar_misc, model_types_misc, model_sizes_misc,
+                gr.State(value=False)],
         outputs=leaderboard_table_misc,
     )
 
     gr.Blocks.load(
         block=demo,
         fn=core.update_df,
-        inputs=[gr.State(value=core.get_available_task_groups(core.get_selected_task_type(2), False)), search_bar_mtbench, langs_bar_mtbench, gr.State(value=[T_SYMBOLS["chat"]]), gr.State(value=False)],
+        inputs=[gr.State(value=core.get_available_task_groups(core.get_selected_task_type(2), False)),
+                search_bar_mtbench, langs_bar_mtbench, gr.State(value=[T_SYMBOLS["chat"]]), gr.State(value=False)],
         outputs=leaderboard_table_mtbench,
     )
 

core.py

@@ -4,7 +4,7 @@ import os
 import numpy as np
 import pandas as pd
 from datasets import load_dataset
-from utils import model_hf_look_up_table_filter
+from utils import add_model_hyperlink
 
 import style
 
@@ -13,7 +13,7 @@ FEW_SHOT_ONLY = ["GSM8K", "TruthfulQA"]
 
 
 def init():
-    global repo_id, config_name, split_name, hidden_df, task_group_names_list, task_group_type_dict, task_groups_shots_dict, languages_list, model_type_dict, mt_bench_language_list
+    global repo_id, config_name, split_name, hidden_df, task_group_names_list, task_group_type_dict, task_groups_shots_dict, languages_list, model_type_dict, mt_bench_language_list, model_link_dict, model_size_dict
 
     repo_id = os.getenv("OGX_LEADERBOARD_DATASET_NAME")
     config_name = os.getenv("OGX_LEADERBOARD_DATASET_CONFIG")
@@ -33,6 +33,14 @@ def init():
     model_type_df = hidden_df[["Model_Name", "Model_Type"]].drop_duplicates()
     model_type_dict = model_type_df.set_index("Model_Name")["Model_Type"].to_dict()
 
+    model_size_df = hidden_df[["Model_Name", "Model_Size"]].drop_duplicates()
+    model_size_df['Model_Size'] = model_size_df['Model_Size'].fillna(0)
+    model_size_dict = model_size_df.set_index("Model_Name")["Model_Size"].to_dict()
+
+    model_link_df = hidden_df[["Model_Name", "Model_Link"]].drop_duplicates()
+    model_link_df["Model_Link"] = model_link_df["Model_Link"].apply(lambda x: f"https://huggingface.co/" + str(x))
+    model_link_dict = model_link_df.set_index("Model_Name")["Model_Link"].to_dict()
+
     hidden_df = hidden_df.pivot_table(
         columns=["Task_Group", "Few_Shot", "Language"],
         index=["Model_Name"],
@@ -43,19 +51,8 @@ def init():
     hidden_df["Type"] = hidden_df["Model_Name"].apply(lambda x: style.T_SYMBOLS[model_type_dict[x]])
 
 
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"> {model_name} </a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/" + model_hf_look_up_table_filter[model_name]['link']
-    return model_hyperlink(link, model_name)
-
-
 def sort_cols(df: pd.DataFrame, fewshot: bool = False) -> pd.DataFrame:
     task_cols = get_task_columns(df)
-    df['Model_Name'] = df['Model_Name'].apply(
-        lambda x: make_clickable_model(x) if x in model_hf_look_up_table_filter else x)
     return df.reindex(["Type", "Model_Name", "Average"] + sorted(task_cols), axis=1)
 
 
@@ -76,6 +73,13 @@ def filter_type(df: pd.DataFrame, model_types: list[str]) -> pd.DataFrame:
     return df[df["Type"].isin(model_types)]
 
 
+def filter_model_size(df: pd.DataFrame, model_sizes, lookup: dict):
+    filtered_model_size = [model_name for model_name, model_size in lookup.items() if
+                           model_sizes[0] <= model_size <= model_sizes[1]]
+    filtered_df = df[df['Model_Name'].isin(filtered_model_size)]
+    return filtered_df
+
+
 def search_model(df: pd.DataFrame, query: str) -> pd.DataFrame:
     """Keep only rows for which model name matches search query"""
     query = query.replace(";", "|")
@@ -134,10 +138,9 @@ def update_df(
     df = filter_type(df, model_types)
 
     if model_sizes:
-        result = [key for key, value in model_hf_look_up_table_filter.items() if
-                  (value.get("model_size") >= model_sizes[0] and value.get("model_size") <= model_sizes[1])]
-        df = df[df['Model_Name'].isin(result)]
+        df = filter_model_size(df=df, model_sizes=model_sizes, lookup=model_size_dict)
 
+    df = add_model_hyperlink(df, model_link_dict)
     if format:
         return sort_cols(df, fewshot).style.format(precision=2, decimal=".", na_rep="N/A")
     else:

utils.py

@@ -1,193 +1,7 @@
-model_hf_look_up_table_filter = {
-    "Aya-23-8B": {
-        "link": "CohereForAI/aya-23-8B",
-        "model_size": 8
-    },
-    "Bloom-7b1": {
-        "link": "bigscience/bloom-7b1",
-        "model_size": 7,
-    },
-    "Bloomz-7b1": {
-        "link": "bigscience/bloomz-7b1",
-        "model_size": 7,
-    },
-    "Meta-Llama-2-7B": {
-        "link": "meta-llama/Llama-2-7b",
-        "model_size": 7,
-    },
-    "Gemma-7b": {
-        "link": "google/gemma-7b",
-        "model_size": 7,
-    },
-    "Gemma-1.1-7b-Instruct": {
-        "link": "google/gemma-1.1-7b-it",
-        "model_size": 7,
-    },
-    "Meta-Llama-3-8B": {
-        "link": "meta-llama/Meta-Llama-3-8B",
-        "model_size": 8
-    },
-    "Meta-Llama-3-8B-Instruct": {
-        "link": "meta-llama/Meta-Llama-3-8B-Instruct",
-        "model_size": 8
-    },
-    "Mistral-7B-Instruct-v0.3": {
-        "link": "mistralai/Mistral-7B-Instruct-v0.3",
-        "model_size": 7
-    },
-    "Mistral-7B-Instruct-v0.1": {
-        "link": "mistralai/Mistral-7B-Instruct-v0.1",
-        "model_size": 7
-    },
-    "Mistral-7B-Instruct-v0.2": {
-        "link": "mistralai/Mistral-7B-Instruct-v0.2",
-        "model_size": 7
-    },
-    "Mistral-7B-v0.1": {
-        "link": "mistralai/Mistral-7B-v0.1",
-        "model_size": 7
-    },
-    "Mistral-7B-v0.3": {
-        "link": "mistralai/Mistral-7B-v0.3",
-        "model_size": 7
-    },
-    "Occiglot-7b-eu5": {
-        "link": "occiglot/occiglot-7b-eu5",
-        "model_size": 7
-    },
-    "Occiglot-7b-eu5-Instruct": {
-        "link": "occiglot/occiglot-7b-eu5-instruct",
-        "model_size": 7
-    },
-    "Phi-3-mini-4k-Instruct": {
-        "link": "microsoft/Phi-3-mini-4k-instruct",
-        "model_size": 3.8
-    },
-    "Qwen2-7B": {
-        "link": "Qwen/Qwen2-7B-Instruct",
-        "model_size": 7
-    },
-    "Qwen2-7B-Instruct": {
-        "link": "Qwen/Qwen2-7B-Instruct",
-        "model_size": 7
-    },
-    "7B_24EU_2.5T_bactrianx17_bb_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_2.5T_bactrianx5_bb_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_2.5T_honey_ckp2701": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_2T_bactrianx17_bb_ckp2": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_2T_bactrianx5_bb_ckp2": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_2.86T_EP5_iter_0681300": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_2.86T_iter_0602100": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_1.45T_bactrianx17_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_1.45T_bactrianx17_bb_ckp2": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_1.45T_bactrianx5_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_1.65T_bactrianx17_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_1.65T_bactrianx17_bb_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_24EU_1.65T_bactrianx5_ckp1": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EN_200B_iter_0047683": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EQUAL_200B_iter_0046950": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_1.1T_iter_0236250": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_1.45T_iter_0346050": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_1.65T_iter_0393075": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_2.5T_DE_213B": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_2.5T_DE_262B": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_2.5T_iter_0602100": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_2T_iter_0477675": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_2T_iter_0477900": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_2T_iter_0478125": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_3T_oscar_iter_0715255": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_3T_fw_iter_0715255": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_fw_3T_honey_ckp1350": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_fw_3.1T_iter_0025875": {
-        "link": "",
-        "model_size": 7
-    },
-    "7B_EU24_1.1T_bactrianx_ckp2": {
-        "link": "",
-        "model_size": 7
-    },
-
-
-
-}
+def add_model_hyperlink(df, lookup):
+    df["Model_Name"] = df["Model_Name"].apply(
+        lambda
+            x: f'<a target="_blank" href="{lookup[x]}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"> {x} </a>' if
+        x in lookup else x
+    )
+    return df