filter by benchmark

app.py

@@ -11,7 +11,9 @@ from src.about import (
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
+    Tasks,
 )
+
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
@@ -59,8 +61,12 @@ try:
 except Exception:
     restart_space()
 
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_DF = get_leaderboard_df(
+    EVAL_RESULTS_PATH,
+    EVAL_REQUESTS_PATH,
+    COLS,
+    BENCHMARK_COLS,
+)
 
 (
     finished_eval_queue_df,
@@ -85,124 +91,176 @@ def init_leaderboard(dataframe):
         filter_columns=[
             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
             ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
         ],
-        bool_checkboxgroup_label="Hide models",
+        # bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
 
 
+task_map = {getattr(Tasks, t).value.col_name: getattr(Tasks, t).name for t in dir(Tasks) if not t.startswith("_")}
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Position Bias Analyzer", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("[ENG] Position Bias Analyzer", elem_id="llm-benchmark-tab-table", id=0):
+            for filter in BENCHMARK_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = True
+
+            FILTERED_COLS = [c for c in BENCHMARK_COLS if not c.startswith("[ENG-P]")]
+            for filter in FILTERED_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = False
+
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("🏅 LLMBar Natural", elem_id="llm-benchmark-tab-table", id=4):
+        with gr.TabItem("[PT] Position Bias Analyzer", elem_id="llm-benchmark-tab-table", id=1):
+            for filter in BENCHMARK_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = True
+
+            FILTERED_COLS = [c for c in BENCHMARK_COLS if not c.startswith("[PT-P]")]
+            for filter in FILTERED_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = False
+
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("🏅 LLMBar Adversarial (Manual)", elem_id="llm-benchmark-tab-table", id=5):
+        with gr.TabItem("[ENG] LLMBar Natural", elem_id="llm-benchmark-tab-table", id=2):
+            for filter in BENCHMARK_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = True
+
+            FILTERED_COLS = [c for c in BENCHMARK_COLS if not c.startswith("[ENG-N]")]
+            for filter in FILTERED_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = False
+
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+        with gr.TabItem("[PT] LLMBar Natural", elem_id="llm-benchmark-tab-table", id=3):
+            for filter in BENCHMARK_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = True
+
+            FILTERED_COLS = [c for c in BENCHMARK_COLS if not c.startswith("[PT-N]")]
+            for filter in FILTERED_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = False
+
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("[ENG] LLMBar Adversarial (Manual)", elem_id="llm-benchmark-tab-table", id=4):
+            for filter in BENCHMARK_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = True
+
+            FILTERED_COLS = [c for c in BENCHMARK_COLS if not c.startswith("[ENG-A]")]
+            for filter in FILTERED_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = False
+
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+        with gr.TabItem("[PT] LLMBar Adversarial (Manual)", elem_id="llm-benchmark-tab-table", id=5):
+            for filter in BENCHMARK_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = True
+
+            FILTERED_COLS = [c for c in BENCHMARK_COLS if not c.startswith("[PT-A]")]
+            for filter in FILTERED_COLS:
+                getattr(AutoEvalColumn, task_map[filter]).displayed_by_default = False
+
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=6):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+        # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        #     with gr.Column():
+        #         with gr.Row():
+        #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+        #         with gr.Column():
+        #             with gr.Accordion(
+        #                 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     finished_eval_table = gr.components.Dataframe(
+        #                         value=finished_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        #             with gr.Accordion(
+        #                 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     running_eval_table = gr.components.Dataframe(
+        #                         value=running_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+
+        #             with gr.Accordion(
+        #                 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+        #                 open=False,
+        #             ):
+        #                 with gr.Row():
+        #                     pending_eval_table = gr.components.Dataframe(
+        #                         value=pending_eval_queue_df,
+        #                         headers=EVAL_COLS,
+        #                         datatype=EVAL_TYPES,
+        #                         row_count=5,
+        #                     )
+        # with gr.Row():
+        #     gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+        # with gr.Row():
+        #     with gr.Column():
+        #         model_name_textbox = gr.Textbox(label="Model name")
+        #         revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+        #         model_type = gr.Dropdown(
+        #             choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+        #             label="Model type",
+        #             multiselect=False,
+        #             value=None,
+        #             interactive=True,
+        #         )
+
+        #     with gr.Column():
+        #         precision = gr.Dropdown(
+        #             choices=[i.value.name for i in Precision if i != Precision.Unknown],
+        #             label="Precision",
+        #             multiselect=False,
+        #             value="float16",
+        #             interactive=True,
+        #         )
+        #         weight_type = gr.Dropdown(
+        #             choices=[i.value.name for i in WeightType],
+        #             label="Weights type",
+        #             multiselect=False,
+        #             value="Original",
+        #             interactive=True,
+        #         )
+        #         base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+        # submit_button = gr.Button("Submit Eval")
+        # submission_result = gr.Markdown()
+        # submit_button.click(
+        #     add_new_eval,
+        #     [
+        #         model_name_textbox,
+        #         base_model_name_textbox,
+        #         revision_name_textbox,
+        #         precision,
+        #         weight_type,
+        #         model_type,
+        #     ],
+        #     submission_result,
+        # )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

src/about.py

@@ -13,8 +13,51 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("eng_natural_vanilla_acc", "acc", "[ENG-N] Vanilla Acc")
+    task1 = Task("eng_natural_vanilla_pa", "pa", "[ENG-N] Vanilla PA")
+    task2 = Task("eng_natural_metrics_reference_acc", "acc", "[ENG-N] Metrics Reference Acc")
+    task3 = Task("eng_natural_metrics_reference_pa", "pa", "[ENG-N] Metrics Reference PA")
+    task4 = Task("eng_natural_swap_acc", "acc", "[ENG-N] Swap Acc")
+    task5 = Task("eng_natural_swap_pa", "pa", "[ENG-N] Swap PA")
+    task6 = Task("eng_natural_swap_cot_acc", "acc", "[ENG-N] Swap COT Acc")
+    task7 = Task("eng_natural_swap_cot_pa", "pa", "[ENG-N] Swap COT PA")
+
+    task8 = Task("eng_adversarial_manual_vanilla_acc", "acc", "[ENG-A] Vanilla Acc")
+    task9 = Task("eng_adversarial_manual_vanilla_pa", "pa", "[ENG-A] Vanilla PA")
+    task10 = Task("eng_adversarial_manual_metrics_reference_acc", "acc", "[ENG-A] Metrics Reference Acc")
+    task11 = Task("eng_adversarial_manual_metrics_reference_pa", "pa", "[ENG-A] Metrics Reference PA")
+    task12 = Task("eng_adversarial_manual_swap_acc", "acc", "[ENG-A] Swap Acc")
+    task13 = Task("eng_adversarial_manual_swap_pa", "pa", "[ENG-A] Swap PA")
+    task14 = Task("eng_adversarial_manual_swap_cot_acc", "acc", "[ENG-A] Swap COT Acc")
+    task15 = Task("eng_adversarial_manual_swap_cot_pa", "pa", "[ENG-A] Swap COT PA")
+
+    task16 = Task("pt_natural_vanilla_acc", "acc", "[PT-N] Vanilla Acc")
+    task17 = Task("pt_natural_vanilla_pa", "pa", "[PT-N] Vanilla PA")
+    task18 = Task("pt_natural_metrics_reference_acc", "acc", "[PT-N] Metrics Reference Acc")
+    task19 = Task("pt_natural_metrics_reference_pa", "pa", "[PT-N] Metrics Reference PA")
+    task20 = Task("pt_natural_swap_acc", "acc", "[PT-N] Swap Acc")
+    task21 = Task("pt_natural_swap_pa", "pa", "[PT-N] Swap PA")
+    task22 = Task("pt_natural_swap_cot_acc", "acc", "[PT-N] Swap COT Acc")
+    task23 = Task("pt_natural_swap_cot_pa", "pa", "[PT-N] Swap COT PA")
+
+    task24 = Task("pt_adversarial_manual_vanilla_acc", "acc", "[PT-A] Vanilla Acc")
+    task25 = Task("pt_adversarial_manual_vanilla_pa", "pa", "[PT-A] Vanilla PA")
+    task26 = Task("pt_adversarial_manual_metrics_reference_acc", "acc", "[PT-A] Metrics Reference Acc")
+    task27 = Task("pt_adversarial_manual_metrics_reference_pa", "pa", "[PT-A] Metrics Reference PA")
+    task28 = Task("pt_adversarial_manual_swap_acc", "acc", "[PT-A] Swap Acc")
+    task29 = Task("pt_adversarial_manual_swap_pa", "pa", "[PT-A] Swap PA")
+    task30 = Task("pt_adversarial_manual_swap_cot_acc", "acc", "[PT-A] Swap COT Acc")
+    task31 = Task("pt_adversarial_manual_swap_cot_pa", "pa", "[PT-A] Swap COT PA")
+
+    task32 = Task("eng_pba_extraction_avg", "avg", "[ENG-P] Extraction Rate")
+    task33 = Task("eng_pba_positional_avg", "avg", "[ENG-P] Positional Consistency Avg")
+    task34 = Task("eng_pba_positional_std", "std", "[ENG-P] Positional Consistency Std")
+    task35 = Task("eng_pba_preference_avg", "avg", "[ENG-P] Preference Score")
+
+    task36 = Task("pt_pba_extraction_avg", "avg", "[PT-P] Extraction Rate")
+    task37 = Task("pt_pba_positional_avg", "avg", "[PT-P] Positional Consistency Avg")
+    task38 = Task("pt_pba_positional_std", "std", "[PT-P] Positional Consistency Std")
+    task39 = Task("pt_pba_preference_avg", "avg", "[PT-P] Preference Score")
 
 
 NUM_FEWSHOT = 0  # Change with your few shot
@@ -26,7 +69,6 @@ TITLE = """<h1 align="center" id="space-title">LLM as Judge Eval</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
 """
 
 # Which evaluations are you running? how can people reproduce what you have?

src/display/utils.py

@@ -5,6 +5,7 @@ import pandas as pd
 
 from src.about import Tasks
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
@@ -20,15 +21,16 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", False)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", False)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
@@ -43,6 +45,7 @@ auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sh
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -53,12 +56,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -83,11 +87,13 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
@@ -100,6 +106,7 @@ class Precision(Enum):
             return Precision.bfloat16
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 
@@ -107,4 +114,3 @@ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/leaderboard/read_evals.py

@@ -14,22 +14,22 @@ from src.submission.check_validity import is_model_on_hub
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
+
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
 
     @classmethod
@@ -70,14 +70,18 @@ class EvalResult:
         results = {}
         for task in Tasks:
             task = task.value
+            # # We average all scores of a given metric (not all metrics are present in all files)
+            # accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            # if accs.size == 0 or any([acc is None for acc in accs]):
+            #     continue
 
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
+            if [v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark.startswith(k)][0]:
 
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+                results[str(task.benchmark)] = [
+                    v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark.startswith(k)
+                ][0] * 100.0
+            else:
+                results[str(task.benchmark)] = 0
 
         return self(
             eval_name=result_key,
@@ -85,10 +89,10 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            precision=precision,
+            revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
-            architecture=architecture
+            architecture=architecture,
         )
 
     def update_with_request_file(self, requests_path):
@@ -105,7 +109,9 @@ class EvalResult:
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(
+                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
+            )
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
@@ -139,17 +145,13 @@ def get_request_file_for_model(requests_path, model_name, precision):
         f"{model_name}_eval_request_*.json",
     )
     request_files = glob.glob(request_files)
-
     # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
+            if req_content["status"] in ["FINISHED"] and req_content["precision"] == precision.split(".")[-1]:
                 request_file = tmp_request_file
     return request_file
 
@@ -174,6 +176,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
+
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
         eval_result.update_with_request_file(requests_path)
@@ -188,9 +191,8 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict()  # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             continue
-
     return results

src/populate.py

@@ -12,13 +12,9 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
-
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
     return df