feat: use dataclass to manage the dataframes

app.py

@@ -65,40 +65,52 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-try:
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception as e:
-    print(f'failed to download')
-    restart_space()
-
-raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/{LATEST_BENCHMARK_VERSION}")
-
-original_df_qa = get_leaderboard_df(
-    raw_data, task='qa', metric=DEFAULT_METRIC_QA)
-original_df_long_doc = get_leaderboard_df(
-    raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
-print(f'raw data: {len(raw_data)}')
-print(f'QA data loaded: {original_df_qa.shape}')
-print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
-
-leaderboard_df_qa = original_df_qa.copy()
+# try:
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+#         token=TOKEN
+#     )
+# except Exception as e:
+#     print(f'failed to download')
+#     restart_space()
+
+from dataclasses import dataclass
+import pandas as pd
+from typing import Optional
+@dataclass
+class LeaderboardDataStore:
+    raw_data: Optional[list]
+    original_df_qa: Optional[pd.DataFrame]
+    original_df_long_doc: Optional[pd.DataFrame]
+    leaderboard_df_qa: Optional[pd.DataFrame]
+    leaderboard_df_long_doc: Optional[pd.DataFrame]
+    reranking_models: Optional[list]
+
+data = {}
+data["AIR-Bench_24.04"] = LeaderboardDataStore(None, None, None, None, None, None)
+data["AIR-Bench_24.04"].raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/AIR-Bench_24.04")
+data["AIR-Bench_24.04"].original_df_qa = get_leaderboard_df(
+    data["AIR-Bench_24.04"].raw_data, task='qa', metric=DEFAULT_METRIC_QA)
+data["AIR-Bench_24.04"].original_df_long_doc = get_leaderboard_df(
+    data["AIR-Bench_24.04"].raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
+print(f'raw data: {len(data["AIR-Bench_24.04"].raw_data)}')
+print(f'QA data loaded: {data["AIR-Bench_24.04"].original_df_qa.shape}')
+print(f'Long-Doc data loaded: {len(data["AIR-Bench_24.04"].original_df_long_doc)}')
+
+data["AIR-Bench_24.04"].leaderboard_df_qa = data["AIR-Bench_24.04"].original_df_qa.copy()
 # leaderboard_df_qa = leaderboard_df_qa[has_no_nan_values(df, _benchmark_cols)]
 shown_columns_qa, types_qa = get_default_cols(
-    'qa', leaderboard_df_qa.columns, add_fix_cols=True)
-leaderboard_df_qa = leaderboard_df_qa[~leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
-leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+    'qa', data["AIR-Bench_24.04"].leaderboard_df_qa.columns, add_fix_cols=True)
+data["AIR-Bench_24.04"].leaderboard_df_qa = data["AIR-Bench_24.04"].leaderboard_df_qa[~data["AIR-Bench_24.04"].leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
+data["AIR-Bench_24.04"].leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-leaderboard_df_long_doc = original_df_long_doc.copy()
+data["AIR-Bench_24.04"].leaderboard_df_long_doc = data["AIR-Bench_24.04"].original_df_long_doc.copy()
 shown_columns_long_doc, types_long_doc = get_default_cols(
-    'long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
-leaderboard_df_long_doc = leaderboard_df_long_doc[~leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
-leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+    'long-doc', data["AIR-Bench_24.04"].leaderboard_df_long_doc.columns, add_fix_cols=True)
+data["AIR-Bench_24.04"].leaderboard_df_long_doc = data["AIR-Bench_24.04"].leaderboard_df_long_doc[~data["AIR-Bench_24.04"].leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+data["AIR-Bench_24.04"].leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-# select reranking model
-reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in raw_data])))
+data["AIR-Bench_24.04"].reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in data["AIR-Bench_24.04"].raw_data])))
 
 
 def update_metric_qa(
@@ -110,7 +122,7 @@ def update_metric_qa(
         show_anonymous: bool,
         show_revision_and_timestamp,
 ):
-    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+    return update_metric(data["AIR-Bench_24.04"].raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 def update_metric_long_doc(
         metric: str,
@@ -121,7 +133,7 @@ def update_metric_long_doc(
         show_anonymous: bool,
         show_revision_and_timestamp,
 ):
-    return update_metric(raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+    return update_metric(data["AIR-Bench_24.04"].raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 
 demo = gr.Blocks(css=custom_css)
@@ -160,10 +172,10 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking models
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(reranking_models)
-                        leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
+                                selected_rerankings = get_reranking_dropdown(data["AIR-Bench_24.04"].reranking_models)
+                        leaderboard_table = get_leaderboard_table(data["AIR-Bench_24.04"].leaderboard_df_qa, types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
+                        hidden_leaderboard_table_for_search = get_leaderboard_table(data["AIR-Bench_24.04"].original_df_qa, types_qa, visible=False)
 
                         set_listeners(
                             "qa",
@@ -198,11 +210,11 @@ with demo:
                                 search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
                                 selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        lb_df_retriever = data["AIR-Bench_24.04"].leaderboard_df_qa[data["AIR-Bench_24.04"].leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
                         lb_df_retriever = reset_rank(lb_df_retriever)
                         lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        hidden_lb_df_retriever = data["AIR-Bench_24.04"].original_df_qa[data["AIR-Bench_24.04"].original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
                         hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
                         hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
 
@@ -234,7 +246,7 @@ with demo:
                             queue=True
                         )
                     with gr.TabItem("Reranking Only", id=12):
-                        lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_df_reranker = data["AIR-Bench_24.04"].leaderboard_df_qa[data["AIR-Bench_24.04"].leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         lb_df_reranker = reset_rank(lb_df_reranker)
                         reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
@@ -243,7 +255,7 @@ with demo:
                             with gr.Column(scale=1):
                                 search_bar_reranker = gr.Textbox(show_label=False, visible=False)
                         lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
-                        hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        hidden_lb_df_reranker = data["AIR-Bench_24.04"].original_df_qa[data["AIR-Bench_24.04"].original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
                         hidden_lb_table_reranker = get_leaderboard_table(
                             hidden_lb_df_reranker, types_qa, visible=False
@@ -301,15 +313,15 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking model
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(reranking_models)
+                                selected_rerankings = get_reranking_dropdown(data["AIR-Bench_24.04"].reranking_models)
 
                         lb_table = get_leaderboard_table(
-                            leaderboard_df_long_doc, types_long_doc
+                            data["AIR-Bench_24.04"].leaderboard_df_long_doc, types_long_doc
                         )
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
                         hidden_lb_table_for_search = get_leaderboard_table(
-                            original_df_long_doc, types_long_doc, visible=False
+                            data["AIR-Bench_24.04"].original_df_long_doc, types_long_doc, visible=False
                         )
 
                         set_listeners(
@@ -345,12 +357,12 @@ with demo:
                                 search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
                                 selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever_long_doc = leaderboard_df_long_doc[
-                            leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        lb_df_retriever_long_doc = data["AIR-Bench_24.04"].leaderboard_df_long_doc[
+                            data["AIR-Bench_24.04"].leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
                         lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
-                        hidden_lb_db_retriever_long_doc = original_df_long_doc[
-                            original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        hidden_lb_db_retriever_long_doc = data["AIR-Bench_24.04"].original_df_long_doc[
+                            data["AIR-Bench_24.04"].original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
                         hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
                         lb_table_retriever_long_doc = get_leaderboard_table(
@@ -386,8 +398,8 @@ with demo:
                             queue=True
                         )
                     with gr.TabItem("Reranking Only", id=22):
-                        lb_df_reranker_ldoc = leaderboard_df_long_doc[
-                            leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        lb_df_reranker_ldoc = data["AIR-Bench_24.04"].leaderboard_df_long_doc[
+                            data["AIR-Bench_24.04"].leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
                             ]
                         lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
                         reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
@@ -397,7 +409,7 @@ with demo:
                             with gr.Column(scale=1):
                                 search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
                         lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
-                        hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        hidden_lb_df_reranker_ldoc = data["AIR-Bench_24.04"].original_df_long_doc[data["AIR-Bench_24.04"].original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
                         hidden_lb_table_reranker_ldoc = get_leaderboard_table(
                             hidden_lb_df_reranker_ldoc, types_long_doc, visible=False

src/envs.py

@@ -27,7 +27,7 @@ BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
 
 BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.04",
-    "AIR-Bench_24.05",
+    # "AIR-Bench_24.05",
 ]
 
 LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]