feat: implement the version selector for qa

app.py

@@ -1,4 +1,5 @@
 import gradio as gr
+import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from src.about import (
@@ -13,7 +14,7 @@ from src.display.css_html_js import custom_css
 from src.envs import (
     API,
     EVAL_RESULTS_PATH,
-    REPO_ID, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST
+    REPO_ID, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST, LATEST_BENCHMARK_VERSION
 )
 from src.loaders import (
     load_eval_results
@@ -48,9 +49,10 @@ def restart_space():
 #     print(f'failed to download')
 #     restart_space()
 
-
+global data
 data = load_eval_results(EVAL_RESULTS_PATH)
-
+global datastore
+datastore = data[LATEST_BENCHMARK_VERSION]
 
 def update_metric_qa(
         metric: str,
@@ -60,28 +62,36 @@ def update_metric_qa(
         query: str,
         show_anonymous: bool,
         show_revision_and_timestamp: bool,
-        selected_version: str,
-):
-    return update_metric(data[selected_version].raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
-
-
-def update_metric_long_doc(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp,
 ):
-    return update_metric(data["AIR-Bench_24.04"].raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
-
-
-DOMAIN_COLS_QA = list(frozenset([c.value.domain for c in list(QABenchmarks)]))
-LANG_COLS_QA = list(frozenset([c.value.lang for c in list(QABenchmarks)]))
-
-DOMAIN_COLS_LONG_DOC = list(frozenset([c.value.domain for c in list(LongDocBenchmarks)]))
-LANG_COLS_LONG_DOC = list(frozenset([c.value.lang for c in list(LongDocBenchmarks)]))
+    return update_metric(datastore, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+
+
+# def update_metric_long_doc(
+#         metric: str,
+#         domains: list,
+#         langs: list,
+#         reranking_model: list,
+#         query: str,
+#         show_anonymous: bool,
+#         show_revision_and_timestamp,
+# ):
+#     return update_metric(datastore.raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+
+
+def update_datastore(version):
+    global datastore
+    global data
+    datastore = data[version]
+    selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug])
+    selected_langs = get_language_dropdown(QABenchmarks[datastore.slug])
+    selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
+    leaderboard_table = get_leaderboard_table(
+        datastore.raw_df_qa, datastore.types_qa)
+    hidden_leaderboard_table_for_search = get_leaderboard_table(
+        datastore.raw_df_qa, datastore.types_qa, visible=False)
+    return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table_for_search
+# DOMAIN_COLS_LONG_DOC = list(frozenset([c.value.domain for c in list(LongDocBenchmarks)]))
+# LANG_COLS_LONG_DOC = list(frozenset([c.value.lang for c in list(LongDocBenchmarks)]))
 
 demo = gr.Blocks(css=custom_css)
 
@@ -99,11 +109,12 @@ with demo:
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
+                            selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug])
+                            # selected_domains = get_domain_dropdown(QABenchmarks["2404"])
                         # select language
                         with gr.Row():
-                            selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
-
+                            selected_langs = get_language_dropdown(QABenchmarks[datastore.slug])
+                            # selected_langs = get_language_dropdown(QABenchmarks["2404"])
                     with gr.Column():
                         # select the metric
                         selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
@@ -119,16 +130,25 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking models
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(data["AIR-Bench_24.04"].reranking_models)
-                        leaderboard_table = get_leaderboard_table(data["AIR-Bench_24.04"].leaderboard_df_qa, data["AIR-Bench_24.04"].types_qa)
+                                selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
+                        #  shown_table
+                        leaderboard_table = get_leaderboard_table(
+                            datastore.leaderboard_df_qa, datastore.types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_leaderboard_table_for_search = get_leaderboard_table(data["AIR-Bench_24.04"].raw_df_qa, data["AIR-Bench_24.04"].types_qa, visible=False)
+                        hidden_leaderboard_table_for_search = get_leaderboard_table(
+                            datastore.raw_df_qa, datastore.types_qa, visible=False)
 
+                        selected_version.change(
+                            update_datastore,
+                            [selected_version,],
+                            [selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table_for_search]
+                        )
                         set_listeners(
                             "qa",
                             leaderboard_table,
                             hidden_leaderboard_table_for_search,
                             search_bar,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings,
@@ -147,7 +167,6 @@ with demo:
                                 search_bar,
                                 show_anonymous,
                                 show_revision_and_timestamp,
-                                selected_version,
                             ],
                             leaderboard_table,
                             queue=True
@@ -468,3 +487,5 @@ if __name__ == "__main__":
     scheduler.start()
     demo.queue(default_concurrency_limit=40)
     demo.launch()
+
+

src/benchmarks.py

@@ -26,33 +26,50 @@ class Benchmark:
 
 
 # create a function return an enum class containing all the benchmarks
-def get_benchmarks_enum(benchmark_version):
-    qa_benchmark_dict = {}
-    long_doc_benchmark_dict = {}
-    for task, domain_dict in BenchmarkTable[benchmark_version].items():
-        for domain, lang_dict in domain_dict.items():
-            for lang, dataset_list in lang_dict.items():
-                if task == "qa":
-                    benchmark_name = f"{domain}_{lang}"
-                    benchmark_name = get_safe_name(benchmark_name)
+def get_benchmarks_enum(benchmark_version, task_type):
+    benchmark_dict = {}
+    if task_type == "qa":
+        for task, domain_dict in BenchmarkTable[benchmark_version].items():
+            if task != task_type:
+                continue
+            for domain, lang_dict in domain_dict.items():
+                for lang, dataset_list in lang_dict.items():
+                    benchmark_name = get_safe_name(f"{domain}_{lang}")
                     col_name = benchmark_name
                     for metric in dataset_list:
-                        qa_benchmark_dict[benchmark_name] = \
-                            Benchmark(
-                                benchmark_name, metric, col_name, domain, lang, task)
-                elif task == "long-doc":
+                        if "test" not in dataset_list[metric]["splits"]:
+                            continue
+                        benchmark_dict[benchmark_name] = \
+                            Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+    elif task_type == "long-doc":
+        for task, domain_dict in BenchmarkTable[benchmark_version].items():
+            if task != task_type:
+                continue
+            for domain, lang_dict in domain_dict.items():
+                for lang, dataset_list in lang_dict.items():
                     for dataset in dataset_list:
                         benchmark_name = f"{domain}_{lang}_{dataset}"
                         benchmark_name = get_safe_name(benchmark_name)
                         col_name = benchmark_name
                         for metric in METRIC_LIST:
-                            long_doc_benchmark_dict[benchmark_name] = \
-                                Benchmark(
-                                    benchmark_name, metric, col_name, domain,
-                                                                            lang, task)
-    return qa_benchmark_dict, long_doc_benchmark_dict
+                            benchmark_dict[benchmark_name] = \
+                                Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+    return benchmark_dict
 
-_qa_benchmark_dict, _long_doc_benchmark_dict = get_benchmarks_enum('AIR-Bench_24.04')
 
-QABenchmarks = Enum('QABenchmarks', _qa_benchmark_dict)
-LongDocBenchmarks = Enum('LongDocBenchmarks', _long_doc_benchmark_dict)
+versions = ("AIR-Bench_24.04", "AIR-Bench_24.05")
+qa_benchmark_dict = {}
+for version in versions:
+    safe_version_name = get_safe_name(version)[-4:]
+    qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_benchmarks_enum(version, "qa"))
+
+long_doc_benchmark_dict = {}
+for version in versions:
+    safe_version_name = get_safe_name(version)[-4:]
+    long_doc_benchmark_dict[safe_version_name] = Enum(f"LongDocBenchmarks_{safe_version_name}", get_benchmarks_enum(version, "long-doc"))
+
+# _qa_benchmark_dict, = get_benchmarks_enum('AIR-Bench_24.04', "qa")
+# _long_doc_benchmark_dict = get_benchmarks_enum('AIR-Bench_24.04', "long-doc")
+
+QABenchmarks = Enum('QABenchmarks', qa_benchmark_dict)
+LongDocBenchmarks = Enum('LongDocBenchmarks', long_doc_benchmark_dict)

src/display/columns.py

@@ -1,16 +1,8 @@
 from dataclasses import dataclass, make_dataclass
 
 from src.benchmarks import QABenchmarks, LongDocBenchmarks
-
-COL_NAME_AVG = "Average ⬆️"
-COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
-COL_NAME_RERANKING_MODEL = "Reranking Model"
-COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
-COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
-COL_NAME_RANK = "Rank 🏆"
-COL_NAME_REVISION = "Revision"
-COL_NAME_TIMESTAMP = "Submission Date"
-COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
+from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
+    COL_NAME_RERANKING_MODEL_LINK, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
 
 def fields(raw_class):
@@ -69,7 +61,7 @@ def get_default_auto_eval_column_dict():
 def make_autoevalcolumn(cls_name, benchmarks):
     auto_eval_column_dict = get_default_auto_eval_column_dict()
     # Leaderboard columns
-    for benchmark in benchmarks:
+    for benchmark in list(benchmarks.value):
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
         )
@@ -78,16 +70,28 @@ def make_autoevalcolumn(cls_name, benchmarks):
     return make_dataclass(cls_name, auto_eval_column_dict, frozen=True)
 
 
-AutoEvalColumnQA = make_autoevalcolumn("AutoEvalColumnQA", QABenchmarks)
-AutoEvalColumnLongDoc = make_autoevalcolumn("AutoEvalColumnLongDoc", LongDocBenchmarks)
+def get_default_col_names_and_types(benchmarks):
+    AutoEvalColumn = make_autoevalcolumn("AutoEvalColumn", benchmarks)
+    col_names = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+    col_types = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+    return col_names, col_types
+
+# AutoEvalColumnQA = make_autoevalcolumn("AutoEvalColumnQA", QABenchmarks)
+# COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+# TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
+
+
+def get_fixed_col_names_and_types():
+    fixed_cols = get_default_auto_eval_column_dict()[:-3]
+    return [c.name for _, _, c in fixed_cols], [c.type for _, _, c in fixed_cols]
+
+# fixed_cols = get_default_auto_eval_column_dict()[:-3]
+# FIXED_COLS = [c.name for _, _, c in fixed_cols]
+# FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
 
-fixed_cols = get_default_auto_eval_column_dict()[:-3]
 
-FIXED_COLS = [c.name for _, _, c in fixed_cols]
-FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
+# AutoEvalColumnLongDoc = make_autoevalcolumn("AutoEvalColumnLongDoc", LongDocBenchmarks)
+# COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+# TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
 
 # Column selection
-COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
-COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
-TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]

src/display/gradio_formatting.py

@@ -1,5 +1,6 @@
 import gradio as gr
 from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
+from src.benchmarks import QABenchmarks
 
 def get_version_dropdown():
     return gr.Dropdown(
@@ -52,7 +53,10 @@ def get_metric_dropdown(metric_list, default_metrics):
     )
 
 
-def get_domain_dropdown(domain_list, default_domains):
+def get_domain_dropdown(benchmarks, default_domains=None):
+    domain_list = list(frozenset([c.value.domain for c in list(benchmarks.value)]))
+    if default_domains is None:
+        default_domains = domain_list
     return gr.CheckboxGroup(
         choices=domain_list,
         value=default_domains,
@@ -61,7 +65,10 @@ def get_domain_dropdown(domain_list, default_domains):
     )
 
 
-def get_language_dropdown(language_list, default_languages):
+def get_language_dropdown(benchmarks, default_languages=None):
+    language_list = list(frozenset([c.value.lang for c in list(benchmarks.value)]))
+    if default_languages is None:
+        default_languages = language_list
     return gr.Dropdown(
         choices=language_list,
         value=default_languages,

src/envs.py

@@ -27,7 +27,7 @@ BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
 
 BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.04",
-    # "AIR-Bench_24.05",
+    "AIR-Bench_24.05",
 ]
 
 LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]
@@ -65,3 +65,12 @@ METRIC_LIST = [
     "mrr_at_100",
     "mrr_at_1000"
 ]
+COL_NAME_AVG = "Average ⬆️"
+COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
+COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
+COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
+COL_NAME_RANK = "Rank 🏆"
+COL_NAME_REVISION = "Revision"
+COL_NAME_TIMESTAMP = "Submission Date"
+COL_NAME_IS_ANONYMOUS = "Anonymous Submission"

src/loaders.py

@@ -3,8 +3,8 @@ from typing import List
 
 import pandas as pd
 
-from src.envs import DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC
-from src.display.columns import COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.envs import DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, COL_NAME_REVISION, COL_NAME_TIMESTAMP, \
+    COL_NAME_IS_ANONYMOUS, BENCHMARK_VERSION_LIST
 
 from src.models import FullEvalResult, LeaderboardDataStore
 from src.utils import get_default_cols, get_leaderboard_df
@@ -50,34 +50,43 @@ def load_raw_eval_results(results_path: str) -> List[FullEvalResult]:
             continue
     return results
 
-
-def load_leaderboard_datastore(file_path) -> LeaderboardDataStore:
-    lb_data_store = LeaderboardDataStore(None, None, None, None, None, None, None, None)
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
+
+def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
+    slug = get_safe_name(version)[-4:]
+    lb_data_store = LeaderboardDataStore(version, slug, None, None, None, None, None, None, None, None)
     lb_data_store.raw_data = load_raw_eval_results(file_path)
     print(f'raw data: {len(lb_data_store.raw_data)}')
 
     lb_data_store.raw_df_qa = get_leaderboard_df(
-        lb_data_store.raw_data, task='qa', metric=DEFAULT_METRIC_QA)
+        lb_data_store, task='qa', metric=DEFAULT_METRIC_QA)
     print(f'QA data loaded: {lb_data_store.raw_df_qa.shape}')
     lb_data_store.leaderboard_df_qa = lb_data_store.raw_df_qa.copy()
-    shown_columns_qa, types_qa = get_default_cols(
-        'qa', lb_data_store.leaderboard_df_qa.columns, add_fix_cols=True)
+    shown_columns_qa, types_qa = get_default_cols('qa', lb_data_store.slug, add_fix_cols=True)
+    # shown_columns_qa, types_qa = get_default_cols(
+    #     'qa', lb_data_store.leaderboard_df_qa.columns, add_fix_cols=True)
     lb_data_store.types_qa = types_qa
     lb_data_store.leaderboard_df_qa = \
         lb_data_store.leaderboard_df_qa[~lb_data_store.leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
     lb_data_store.leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-    lb_data_store.raw_df_long_doc = get_leaderboard_df(
-        lb_data_store.raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
-    print(f'Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}')
-    lb_data_store.leaderboard_df_long_doc = lb_data_store.raw_df_long_doc.copy()
-    shown_columns_long_doc, types_long_doc = get_default_cols(
-        'long-doc', lb_data_store.leaderboard_df_long_doc.columns, add_fix_cols=True)
-    lb_data_store.types_long_doc = types_long_doc
-    lb_data_store.leaderboard_df_long_doc = \
-        lb_data_store.leaderboard_df_long_doc[
-            ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
-    lb_data_store.leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+    # lb_data_store.raw_df_long_doc = get_leaderboard_df(
+    #     lb_data_store, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
+    # print(f'Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}')
+    # lb_data_store.leaderboard_df_long_doc = lb_data_store.raw_df_long_doc.copy()
+    # shown_columns_long_doc, types_long_doc = get_default_cols(
+    #     'long-doc', lb_data_store.leaderboard_df_long_doc.columns, add_fix_cols=True)
+    # lb_data_store.types_long_doc = types_long_doc
+    # lb_data_store.leaderboard_df_long_doc = \
+    #     lb_data_store.leaderboard_df_long_doc[
+    #         ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+    # lb_data_store.leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
     lb_data_store.reranking_models = sorted(
         list(frozenset([eval_result.reranking_model for eval_result in lb_data_store.raw_data])))
@@ -86,8 +95,8 @@ def load_leaderboard_datastore(file_path) -> LeaderboardDataStore:
 
 def load_eval_results(file_path: str):
     output = {}
-    versions = ("AIR-Bench_24.04",)
-    for version in versions:
+    # versions = BENCHMARK_VERSION_LIST
+    for version in BENCHMARK_VERSION_LIST:
         fn = f"{file_path}/{version}"
-        output[version] = load_leaderboard_datastore(fn)
+        output[version] = load_leaderboard_datastore(fn, version)
     return output

src/models.py

@@ -6,7 +6,7 @@ from typing import List, Optional
 import pandas as pd
 
 from src.benchmarks import get_safe_name
-from src.display.columns import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
+from src.envs import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
     COL_NAME_RERANKING_MODEL_LINK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 from src.display.formatting import make_clickable_model
 
@@ -128,6 +128,8 @@ class FullEvalResult:
 
 @dataclass
 class LeaderboardDataStore:
+    version: str
+    slug: str
     raw_data: Optional[list]
     raw_df_qa: Optional[pd.DataFrame]
     raw_df_long_doc: Optional[pd.DataFrame]

src/utils.py

@@ -2,17 +2,14 @@ import json
 import hashlib
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import List
 
 import pandas as pd
 
 from src.benchmarks import QABenchmarks, LongDocBenchmarks
 from src.display.formatting import styled_message, styled_error
-from src.display.columns import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, \
-    COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS, COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, \
-    FIXED_COLS, FIXED_COLS_TYPES
-from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION
-from src.models import FullEvalResult
+from src.display.columns import get_default_col_names_and_types, get_fixed_col_names_and_types
+from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION, COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, \
+    COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
 import re
 
@@ -62,61 +59,95 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[COL_NAME_RETRIEVAL_MODEL].str.contains(query, case=False))]
 
 
-def get_default_cols(task: str, columns: list=[], add_fix_cols: bool=True) -> list:
+def get_default_cols(task: str, version_slug, add_fix_cols: bool=True) -> tuple:
     cols = []
     types = []
     if task == "qa":
-        cols_list = COLS_QA
-        types_list = TYPES_QA
-        benchmark_list = [c.value.col_name for c in list(QABenchmarks)]
-    elif task == "long-doc":
-        cols_list = COLS_LONG_DOC
-        types_list = TYPES_LONG_DOC
-        benchmark_list = [c.value.col_name for c in list(LongDocBenchmarks)]
+        benchmarks = QABenchmarks[version_slug]
+        cols_list, types_list = get_default_col_names_and_types(benchmarks)
+        # cols_list = COLS_QA
+        # types_list = TYPES_QA
+        benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
     else:
         raise NotImplemented
     for col_name, col_type in zip(cols_list, types_list):
         if col_name not in benchmark_list:
             continue
-        if len(columns) > 0 and col_name not in columns:
-            continue
         cols.append(col_name)
         types.append(col_type)
 
     if add_fix_cols:
         _cols = []
         _types = []
+        fixed_cols, fixed_cols_types = get_fixed_col_names_and_types()
         for col_name, col_type in zip(cols, types):
-            if col_name in FIXED_COLS:
+            if col_name in fixed_cols:
                 continue
             _cols.append(col_name)
             _types.append(col_type)
-        cols = FIXED_COLS + _cols
-        types = FIXED_COLS_TYPES + _types
+        cols = fixed_cols + _cols
+        types = fixed_cols_types + _types
     return cols, types
 
 
+# def get_default_cols(task: str, columns: list, add_fix_cols: bool=True) -> list:
+#     cols = []
+#     types = []
+#     if task == "qa":
+#         cols_list = COLS_QA
+#         types_list = TYPES_QA
+#         benchmark_list = [c.value.col_name for c in list(QABenchmarks)]
+#     elif task == "long-doc":
+#         cols_list = COLS_LONG_DOC
+#         types_list = TYPES_LONG_DOC
+#         benchmark_list = [c.value.col_name for c in list(LongDocBenchmarks)]
+#     else:
+#         raise NotImplemented
+#     for col_name, col_type in zip(cols_list, types_list):
+#         if col_name not in benchmark_list:
+#             continue
+#         if len(columns) > 0 and col_name not in columns:
+#             continue
+#         cols.append(col_name)
+#         types.append(col_type)
+#
+#     if add_fix_cols:
+#         _cols = []
+#         _types = []
+#         for col_name, col_type in zip(cols, types):
+#             if col_name in FIXED_COLS:
+#                 continue
+#             _cols.append(col_name)
+#             _types.append(col_type)
+#         cols = FIXED_COLS + _cols
+#         types = FIXED_COLS_TYPES + _types
+#     return cols, types
+
+
 def select_columns(
         df: pd.DataFrame,
         domain_query: list,
         language_query: list,
         task: str = "qa",
-        reset_ranking: bool = True
+        reset_ranking: bool = True,
+        version_slug: str = None
 ) -> pd.DataFrame:
-    cols, _ = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
+    cols, _ = get_default_cols(task=task, version_slug=version_slug, add_fix_cols=False)
     selected_cols = []
     for c in cols:
         if task == "qa":
-            eval_col = QABenchmarks[c].value
+            eval_col = QABenchmarks[version_slug].value[c].value
         elif task == "long-doc":
-            eval_col = LongDocBenchmarks[c].value
+            eval_col = LongDocBenchmarks[version_slug].value[c].value
         if eval_col.domain not in domain_query:
             continue
         if eval_col.lang not in language_query:
             continue
         selected_cols.append(c)
     # We use COLS to maintain sorting
-    filtered_df = df[FIXED_COLS + selected_cols]
+    fixed_cols, _ = get_fixed_col_names_and_types()
+    filtered_df = df[fixed_cols + selected_cols]
+    filtered_df.replace({"": pd.NA}, inplace=True)
     if reset_ranking:
         filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
         filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
@@ -125,9 +156,17 @@ def select_columns(
 
     return filtered_df
 
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
 
 def _update_table(
         task: str,
+        version: str,
         hidden_df: pd.DataFrame,
         domains: list,
         langs: list,
@@ -137,33 +176,24 @@ def _update_table(
         reset_ranking: bool = True,
         show_revision_and_timestamp: bool = False
 ):
+    version_slug = get_safe_name(version)[-4:]
+    if isinstance(hidden_df, str):
+        print(f"task: {task}")
+        print(f"version: {version}")
+        print(f"hidden_df is a string: {hidden_df}")
     filtered_df = hidden_df.copy()
     if not show_anonymous:
         filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
     filtered_df = filter_models(filtered_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
-    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking)
+    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking, version_slug)
     if not show_revision_and_timestamp:
         filtered_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
     return filtered_df
 
 
-def update_table(
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True
-):
-    return _update_table(
-        "qa",
-        hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
-
-
 def update_table_long_doc(
+        version: str,
         hidden_df: pd.DataFrame,
         domains: list,
         langs: list,
@@ -176,11 +206,12 @@ def update_table_long_doc(
 ):
     return _update_table(
         "long-doc",
+        version,
         hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
 
 
 def update_metric(
-        raw_data: List[FullEvalResult],
+        datastore,
         task: str,
         metric: str,
         domains: list,
@@ -190,9 +221,12 @@ def update_metric(
         show_anonymous: bool = False,
         show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
+    # raw_data = datastore.raw_data
     if task == 'qa':
-        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        leaderboard_df = get_leaderboard_df(datastore, task=task, metric=metric)
+        version = datastore.version
         return update_table(
+            version,
             leaderboard_df,
             domains,
             langs,
@@ -202,8 +236,10 @@ def update_metric(
             show_revision_and_timestamp
         )
     elif task == "long-doc":
-        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        leaderboard_df = get_leaderboard_df(datastore, task=task, metric=metric)
+        version = datastore.version
         return update_table_long_doc(
+            version,
             leaderboard_df,
             domains,
             langs,
@@ -321,17 +357,20 @@ def reset_rank(df):
     return df
 
 
-def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -> pd.DataFrame:
+def get_leaderboard_df(datastore, task: str, metric: str) -> pd.DataFrame:
     """
     Creates a dataframe from all the individual experiment results
     """
+    raw_data = datastore.raw_data
     cols = [COL_NAME_IS_ANONYMOUS, ]
     if task == "qa":
-        cols += COLS_QA
-        benchmark_cols = [t.value.col_name for t in QABenchmarks]
-    elif task == "long-doc":
-        cols += COLS_LONG_DOC
-        benchmark_cols = [t.value.col_name for t in LongDocBenchmarks]
+        benchmarks = QABenchmarks[datastore.slug]
+        cols_qa, _ = get_default_col_names_and_types(benchmarks)
+        cols += cols_qa
+        benchmark_cols = [t.value.col_name for t in list(benchmarks.value)]
+    # elif task == "long-doc":
+    #     cols += COLS_LONG_DOC
+    #     benchmark_cols = [t.value.col_name for t in LongDocBenchmarks]
     else:
         raise NotImplemented
     all_data_json = []
@@ -366,6 +405,7 @@ def set_listeners(
         target_df,
         source_df,
         search_bar,
+        version,
         selected_domains,
         selected_langs,
         selected_rerankings,
@@ -385,11 +425,27 @@ def set_listeners(
         search_bar,
         show_anonymous
     ]
-    search_bar_args = [source_df,] + selector_list
-    selector_args = search_bar_args + [show_revision_and_timestamp,]
+    search_bar_args = [source_df, version,] + selector_list
+    selector_args = [version, source_df] + selector_list + [show_revision_and_timestamp,]
     # Set search_bar listener
     search_bar.submit(update_table_func, search_bar_args, target_df)
 
     # Set column-wise listener
     for selector in selector_list:
         selector.change(update_table_func, selector_args, target_df, queue=True,)
+
+def update_table(
+        version: str,
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp: bool = False,
+        reset_ranking: bool = True,
+):
+    return _update_table(
+        "qa",
+        version,
+        hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)

tests/src/test_benchmarks.py

@@ -2,9 +2,14 @@ from src.benchmarks import QABenchmarks, LongDocBenchmarks
 
 
 def test_qabenchmarks():
-    print(list(QABenchmarks))
-    for benchmark in list(QABenchmarks):
-        print(benchmark.name, benchmark.metric, benchmark.col_name, benchmark.domain, benchmark.lang, benchmark.task)
+    for benchmark_list in list(QABenchmarks):
+        print(benchmark_list.name)
+        for b in list(benchmark_list.value):
+            print(b)
+    qa_benchmarks = QABenchmarks["2404"]
+    l = list(frozenset([c.value.domain for c in list(qa_benchmarks.value)]))
+    print(l)
+
 
 
 def test_longdocbenchmarks():

tests/test_utils.py

@@ -1,9 +1,10 @@
 import pandas as pd
 import pytest
 
-from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols, update_table
-from src.display.columns import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, \
-    COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols
+from app import update_table
+from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, \
+    COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
 
 @pytest.fixture