Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,19 @@
+---
+title: 'Leaderboard: Physical Reasoning from Video'
+emoji: 🏃
+colorFrom: blue
+colorTo: indigo
+sdk: gradio
+sdk_version: 5.23.1
+app_file: app.py
+pinned: false
+license: mit
+hf_oauth: true
+---
+
+# Local debugging
+
+Run `python app.py` and you'll get a gradio link to view the leaderboard.
+
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,945 @@
+import copy
+import datetime
+import json
+import os
+from email.utils import parseaddr
+import re
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from datasets import Dataset, DatasetDict, VerificationMode, get_dataset_config_names, load_dataset
+from huggingface_hub import HfApi
+
+from content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    INTRODUCTION_TEXT,
+    SUBMISSION_TEXT,
+    TITLE,
+    format_error,
+    format_log,
+    format_warning,
+    model_hyperlink,
+)
+
+TOKEN = os.environ.get("HF_TOKEN", None)
+
+
+OWNER = "facebook"
+## private datasets
+SUBMISSION_DATASET = f"{OWNER}/pwm_leaderboard_submissions_internal"
+CONTACT_DATASET = f"{OWNER}/pwm_leaderboard_contact_info_internal"
+## public datasets
+RESULTS_DATASET = f"{OWNER}/pwm_leaderboard_results_public"
+LEADERBOARD_PATH = f"{OWNER}/pwm_leaderboard"
+DATA_VERSION = "1.0.0"
+
+# Dataset paths
+MVP_DATASET = "facebook/minimal_video_pairs"
+INTP_DATASET = "facebook/IntPhys2_test"
+WMQA_DATASET = "facebook/CausalVQA"
+
+# Dataset names
+MVP_NAME = "MVPBench"
+INTP_NAME = "IntPhys 2"
+WMQA_NAME = "CausalVQA"
+
+# Dataset keys
+MVP_KEY = "mvp"
+MVP_MINI_KEY = "mvp_mini"
+INTP_KEY = "intphys2"
+WMQA_KEY = "causalvqa"
+
+TASKS = [
+    (INTP_KEY, INTP_NAME),
+    (MVP_KEY, MVP_NAME),
+    (WMQA_KEY, WMQA_NAME),
+]
+VISIBLE_TASKS = copy.deepcopy(TASKS)
+PRE_COL_NAMES = ["Model Name"]
+POST_COL_NAMES = ["Model Type", "Vision Backbone", "LLM Backbone", "Submission Date"]
+
+
+api = HfApi()
+
+os.makedirs("scored", exist_ok=True)
+
+LOCAL_DEBUG = False
+
+# Display the results
+
+LDB_TEXT_KEYS = ["model", "model_type", "vision_backbone", "llm_backbone"]
+LDB_TEXT_TYPES = ["markdown", "text", "text", "text"]
+MISSING_VALUE = -1.0
+
+HUMAN_BASELINES = {
+    "url": "",
+    "model": "Human",
+    "model_type": "Human",
+    "system_prompt": "test",
+    "vision_backbone": " - ",
+    "llm_backbone": " - ",
+    "num_frames": -1,
+    f"score_{INTP_KEY}": 92.44,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 92.9,
+    f"score_{WMQA_KEY}": 84.78,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+
+GEMINI2_5 = {
+    "url": "https://deepmind.google/models/gemini/flash/",
+    "model": "Gemini 2.5 Flash",
+    "model_type": "Closed",
+    "system_prompt": "test",
+    "vision_backbone": " - ",
+    "llm_backbone": " - ",
+    "num_frames": 10,
+    f"score_{INTP_KEY}": 56.1,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": MISSING_VALUE,
+    f"score_{WMQA_KEY}": 61.66,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+GPT4O = {
+    "url": "https://openai.com/index/gpt-4o-system-card/",
+    "model": "GPT-4o",
+    "model_type": "Closed",
+    "system_prompt": "test",
+    "vision_backbone": " - ",
+    "llm_backbone": " - ",
+    "num_frames": 10,
+    f"score_{INTP_KEY}": 53.19,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 32.5,
+    f"score_{WMQA_KEY}": 50.95,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+INTERN_VL = {
+    "url": "https://internvl.github.io/blog/2024-12-05-InternVL-2.5/",
+    "model": "InternVL2.5",
+    "model_type": "Open",
+    "system_prompt": "test",
+    "vision_backbone": "InternViT-300M",
+    "llm_backbone": "InternLM2.5-7B-Chat",
+    "num_frames": 16,
+    f"score_{INTP_KEY}": MISSING_VALUE,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 39.9,
+    f"score_{WMQA_KEY}": 47.54,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+LLAVA = {
+    "url": "https://huggingface.co/lmms-lab/llava-onevision-qwen2-7b-ov",
+    "model": "LLaVA-OneVision",
+    "model_type": "Open",
+    "system_prompt": "test",
+    "vision_backbone": "SigLIP",
+    "llm_backbone": "Qwen2-7B",
+    "num_frames": 16,
+    f"score_{INTP_KEY}": MISSING_VALUE,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 20.7,
+    f"score_{WMQA_KEY}": 45.27,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+PLM = {
+    "url": "https://github.com/facebookresearch/perception_models",
+    "model": "Perception Language Model (PLM)",
+    "model_type": "Open",
+    "system_prompt": "test",
+    "vision_backbone": "PE",
+    "llm_backbone": "Llama3.1 8B",
+    "num_frames": 16,
+    f"score_{INTP_KEY}": MISSING_VALUE,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 39.7,
+    f"score_{WMQA_KEY}": 50.06,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+QWENVL = {
+    "url": "https://github.com/QwenLM/Qwen2.5-VL",
+    "model": "Qwen2.5-VL",
+    "model_type": "Open",
+    "system_prompt": "test",
+    "vision_backbone": "ViT",
+    "llm_backbone": "Qwen2.5-7B-Instruct",
+    "num_frames": 16,
+    f"score_{INTP_KEY}": 49.12,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 36.7,
+    f"score_{WMQA_KEY}": 49.05,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+GEMINI1_5 = {
+    "url": "https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/1-5-pro",
+    "model": "Gemini 1.5 Pro",
+    "model_type": "Closed",
+    "system_prompt": "test",
+    "vision_backbone": " - ",
+    "llm_backbone": " - ",
+    "num_frames": -1,
+    f"score_{INTP_KEY}": 52.1,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 29.6,
+    f"score_{WMQA_KEY}": MISSING_VALUE,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+VJEPA2 = {
+    "url": "https://ai.meta.com/vjepa/",
+    "model": "V-JEPA 2",
+    "model_type": "Open",
+    "system_prompt": "test",
+    "vision_backbone": "VJEPA 2",
+    "llm_backbone": "Llama3.1 8B",
+    "num_frames": -1,
+    f"score_{INTP_KEY}": 56.4,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": 44.5,
+    f"score_{WMQA_KEY}": 38.99,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+COSMOS = {
+    "url": "https://huggingface.co/nvidia/Cosmos-1.0-Autoregressive-4B",
+    "model": "Cosmos-4B",
+    "model_type": "Open",
+    "system_prompt": "test",
+    "vision_backbone": " - ",
+    "llm_backbone": " - ",
+    "num_frames": -1,
+    f"score_{INTP_KEY}": 48.84,
+    f"score_{MVP_KEY}": MISSING_VALUE,
+    f"score_{MVP_MINI_KEY}": MISSING_VALUE,
+    f"score_{WMQA_KEY}": MISSING_VALUE,
+    "date": "2025-06-11",
+    "organization": "Meta",
+    "submitted_by": "user",
+}
+
+
+def get_dataframe_from_results(eval_results, split):
+    local_df = eval_results[split]
+    local_df = local_df.map(lambda row: {"model": model_hyperlink(row["url"], row["model"])})
+    local_df = local_df.remove_columns(["system_prompt"])#, "url"])
+
+    df = pd.DataFrame(local_df)
+    # reformat the data to keep a single row for a given model and organization pair
+    # in case of multiple entries, choose the ones with latest values
+    df["model_org"] = df["model"].str.cat(df["organization"], sep="-")
+    ldb_m2r = {}
+    for i, row in df.iterrows():
+        if row["model_org"] not in ldb_m2r:
+            ldb_m2r[row["model_org"]] = {}
+
+        prev_d = ldb_m2r[row["model_org"]]
+        new_d = {}
+        for key in LDB_TEXT_KEYS:
+            new_d[key] = row[key] if len(row[key]) > 0 else prev_d.get(key, "NA")
+        for tname, _ in TASKS:
+            new_d[f"score_{tname}"] = (
+                row[f"score_{tname}"] if row[f"score_{tname}"] >= 0 else prev_d.get(f"score_{tname}", MISSING_VALUE)
+            )
+            if tname == "mvp":
+                new_d[f"score_mvp_mini"] = (
+                    row[f"score_mvp_mini"]
+                    if row[f"score_mvp_mini"] >= 0
+                    else prev_d.get(f"score_mvp_mini", MISSING_VALUE)
+                )
+        new_d["date"] = row["date"]
+        ldb_m2r[row["model_org"]] = new_d
+
+    # add Human baseline
+    ldb_m2r["human"] = HUMAN_BASELINES
+    ldb_m2r["gemini2.5"] = GEMINI2_5
+    ldb_m2r["gemini1.5"] = GEMINI1_5
+    ldb_m2r["gpt4o"] = GPT4O
+    ldb_m2r["internvl"] = INTERN_VL
+    ldb_m2r["llavaov"] = LLAVA
+    ldb_m2r["plm"] = PLM
+    ldb_m2r["qwen2.5"] = QWENVL
+    ldb_m2r["vjepa2"] = VJEPA2
+    ldb_m2r["cosmos"] = COSMOS
+    # compute average and convert back to rows
+    ldb_rows = []
+    for key, val in ldb_m2r.items():
+        print(ldb_m2r[key])
+        if "url" in ldb_m2r[key].keys() and ldb_m2r[key]["url"] != "":
+            ldb_m2r[key]["model"] = model_hyperlink(ldb_m2r[key]["url"],ldb_m2r[key]["model"])
+        row = copy.deepcopy(val)
+        score_keys = {k for k in val if k.startswith("score_")}
+        row["score"] = np.round(np.mean([row[sk] for sk in score_keys if (row[sk] != MISSING_VALUE and row[sk] != "-")]), 2)
+        tasks_completed = 0
+        for sk in score_keys:
+            if row[sk] == MISSING_VALUE:
+                row[sk] = "-"
+            else:
+                tasks_completed += 1
+        row["tasks_completed"] = tasks_completed
+        ldb_rows.append(row)
+
+    df = pd.DataFrame(ldb_rows)
+    df = df.query('date >= "2025-06-11"')
+    # df = df.map(lambda row: {"model": model_hyperlink(row["url"], row["model"])})
+
+    # sort
+    df = df.sort_values(by=["tasks_completed", "score"], ascending=False)
+
+    # format numerics
+    numeric_cols = [c for c in df.columns if c.startswith("score_")]
+    for nc in numeric_cols:
+        df[nc] = df[nc].apply(lambda x: np.round(x, 2) if type(x) == float else x)
+
+    # remove columns and rename
+    df.drop(["tasks_completed"], axis=1, inplace=True)
+    col_mapper = {f"score_{tname}": f"{tdisplay} (%)" for tname, tdisplay in TASKS if tname != "mvp"}
+    col_mapper.update(
+        {
+            "model": "Model Name",
+            "model_type": "Model Type",
+            "vision_backbone": "Vision Backbone",
+            "llm_backbone": "LLM Backbone",
+            #"score": "Average Score (%)",
+            "date": "Submission Date",
+        }
+    )
+    df.rename(col_mapper, axis=1, inplace=True)
+    
+    df[f"{MVP_NAME} (%)"] = df.score_mvp_mini.astype(str)
+    df.drop([f"score_{MVP_KEY}", f"score_{MVP_MINI_KEY}"], axis=1, inplace=True)
+    # order columns
+    df = df[PRE_COL_NAMES + [f"{t[1]} (%)" for t in VISIBLE_TASKS] + POST_COL_NAMES]
+    
+    return df
+
+
+def create_dummy_data():
+    # Dummy evals data
+    rows = [
+            {
+            "url": "https://deepmind.google/models/gemini/flash/",
+            "model": "Gemini Test",
+            "model_type": "Closed",
+            "system_prompt": "test",
+            "vision_backbone": " - ",
+            "llm_backbone": " - ",
+            "num_frames": 10,
+            f"score_{INTP_KEY}": 56.1,
+            f"score_{MVP_KEY}": MISSING_VALUE,
+            f"score_{MVP_MINI_KEY}": MISSING_VALUE,
+            f"score_{WMQA_KEY}": 61.66,
+            "date": datetime.datetime.today().strftime("%Y-%m-%d"),
+            "organization": "test",
+            "submitted_by": "octocat",
+            },
+        {
+            "url": "https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf",
+            "model": "Llava 1.6",
+            "model_type": "Open",
+            "system_prompt": "test",
+            "vision_backbone": "CLIP",
+            "llm_backbone": "Mistral",
+            "num_frames": 16,
+            f"score_{INTP_KEY}": MISSING_VALUE,
+            f"score_{MVP_KEY}": MISSING_VALUE,
+            f"score_{MVP_MINI_KEY}": MISSING_VALUE,
+            f"score_{WMQA_KEY}": MISSING_VALUE,
+            "date": datetime.datetime.today().strftime("%Y-%m-%d"),
+            "organization": "test",
+            "submitted_by": "octocat",
+        },
+        {
+            "url": "https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf",
+            "model": "Llava 1.6",
+            "model_type": "Open",
+            "system_prompt": "test",
+            "vision_backbone": "CLIP",
+            "llm_backbone": "Mistral",
+            "num_frames": 16,
+            f"score_{INTP_KEY}": 0.0,
+            f"score_{MVP_KEY}": MISSING_VALUE,
+            f"score_{MVP_MINI_KEY}": MISSING_VALUE,
+            f"score_{WMQA_KEY}": 0.0,
+            "date": datetime.datetime.today().strftime("%Y-%m-%d"),
+            "organization": "test",
+            "submitted_by": "octocat",
+        },
+    ]
+    dt = DatasetDict({"valid": Dataset.from_list(rows), "test": Dataset.from_list(rows)})
+    # Dummy contact
+    contact_info = {
+        "model": "llama",
+        "url": "test",
+        "organization": "test",
+        "username": "test",
+        "mail": "test",
+        "date": datetime.datetime.today().strftime("%Y-%m-%d"),
+    }
+    cdt = DatasetDict({"valid": Dataset.from_list([contact_info]), "test": Dataset.from_list([contact_info])})
+    return dt, cdt
+
+
+DUMMY_DATA = False
+
+
+def get_eval_data():
+    if DUMMY_DATA:
+        eval_results, _ = create_dummy_data()
+    else:
+        eval_results = load_dataset(
+            RESULTS_DATASET,
+            token=TOKEN,
+            download_mode="force_redownload",
+            verification_mode=VerificationMode.NO_CHECKS,
+            trust_remote_code=True,
+        )
+    eval_dataframe_val = get_dataframe_from_results(eval_results=eval_results, split="valid")
+    eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
+    return eval_results, eval_dataframe_val, eval_dataframe_test
+
+
+def restart_space():
+    api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
+
+
+# --- MVP Functions __
+
+
+def validate_mvp(submission_df, split="valid"):
+    subsets = submission_df.data_name.unique()
+    for subset in subsets:
+        assert subset in [MVP_KEY, MVP_MINI_KEY], format_error(
+            f"Wrong tasks, got {subset} but expecting either mvp or mvp_mini"
+        )
+    gold_tasks = get_dataset_config_names(MVP_DATASET, token=TOKEN)
+    for subset in subsets:
+        tasks = submission_df[submission_df.data_name == subset].task.unique()
+        assert len(tasks) == len(gold_tasks), format_error(
+            f"{MVP_NAME} submission must have all tasks, found = {tasks}, expecting = {gold_tasks}"
+        )
+        for task in tasks:
+            sub_df = submission_df[(submission_df.data_name == subset) & (submission_df.task == task)].copy()
+            assert task in gold_tasks, format_error(f"Found unknown task {task} for {MVP_NAME}, check submission")
+            gold_dataset = load_dataset(MVP_DATASET, task, split="full" if subset == MVP_KEY else "mini", token=TOKEN)
+            assert len(sub_df) == len(gold_dataset), format_error(
+                f"Number of examples do not match in user submission, found {len(sub_df)} but expecting {len(gold_dataset)} for task {task} in split {subset}"
+            )
+            id2answer = {row["video_id"]: row["answer"] for row in gold_dataset}
+            for i, r in sub_df.iterrows():
+                assert r["row_id"] in id2answer, format_error(
+                    f"Submission contains row_id {r['row_id']} which doesn't match the dataset's video_id"
+                )
+
+
+def compute_scores_mvp(submission_df, split="valid"):
+    gold_tasks = get_dataset_config_names(MVP_DATASET, token=TOKEN)
+    subsets = submission_df.data_name.unique()
+    scored_subs = []
+    for subset in subsets:
+        tasks = submission_df[submission_df.data_name == subset].task.unique()
+        assert len(tasks) == len(gold_tasks), format_error(f"{MVP_NAME} submission must have all tasks")
+        for task in tasks:
+            sub_df = submission_df[(submission_df.data_name == subset) & (submission_df.task == task)].copy()
+            gold_dataset = load_dataset(MVP_DATASET, task, split="full" if subset == MVP_KEY else "mini", token=TOKEN)
+            id2answer = {row["video_id"]: row["answer"] for row in gold_dataset}
+            correct = []
+            for i, r in sub_df.iterrows():
+                gold_answer = id2answer[r["row_id"]]
+                model_answer = r["model_answer"]
+                if gold_answer == model_answer:
+                    correct.append(1)
+                else:
+                    correct.append(0)
+            sub_df["rating"] = correct
+            scored_subs.append(sub_df)
+    return pd.concat(scored_subs)
+
+
+def aggregate_scores_mvp(scored_submission_df, split="valid"):
+    subsets = scored_submission_df.data_name.unique()
+    subset_scores = {f"score_{s}": 0 for s in subsets}
+    for subset in subsets:
+        tasks = scored_submission_df[scored_submission_df.data_name == subset].task.unique()
+        task_pair_accuracies = []
+        for task in tasks:
+            sub_df = scored_submission_df[
+                (scored_submission_df.data_name == subset) & (scored_submission_df.task == task)
+            ].copy()
+            result_by_vid = {}
+            pair_correct_count = 0
+            for i, row in sub_df.iterrows():
+                video_id = "_".join(row["row_id"].split("_")[:-1])
+                if video_id not in result_by_vid:
+                    result_by_vid[video_id] = [row.to_dict()]
+                else:
+                    result_by_vid[video_id].append(row.to_dict())
+            for video_id, answer_dict_pair in result_by_vid.items():
+                answer_dict_1, answer_dict_2 = answer_dict_pair
+                if answer_dict_1["rating"] == 1 and answer_dict_2["rating"] == 1:
+                    pair_correct_count += 1
+
+            task_pair_accuracies.append((pair_correct_count / len(result_by_vid)) * 100)
+        # compute macro scores
+        subset_scores[f"score_{subset}"] = np.mean(task_pair_accuracies)
+    return subset_scores
+
+
+# --- CausalVQA functions ---
+
+def validate_causalvqa(submission_df, split="test"):
+    #assert split == "test", format_error(f"Split {split} not available for dataset {WMQA_NAME}")
+    split = "train"
+    subsets = submission_df.data_name.unique()
+    for subset in subsets:
+        assert subset in [WMQA_KEY], format_error(
+            f"Wrong tasks, got {subset} but expecting causalvqa"
+        )
+    gold_tasks = get_dataset_config_names(WMQA_DATASET, token=TOKEN)
+    for subset in subsets:
+        tasks = "default"#submission_df[submission_df.data_name == subset].task.unique()
+        sub_df = submission_df[(submission_df.data_name == subset)].copy()
+        gold_dataset = load_dataset(WMQA_DATASET, "", split="train", token=TOKEN) #note, causalvqa only has a test dataset under hf split 'valid'
+        assert len(sub_df) == len(gold_dataset), format_error(
+        f"Number of examples do not match in user submission, found {len(sub_df)} but expecting {len(gold_dataset)} for task {task} in split {subset}"
+            )
+        id2answer = {row["id"]+'_'+str(row["n"]): row["answer"] for row in gold_dataset}
+        for i, r in sub_df.iterrows():
+            assert r["row_id"] in id2answer, format_error(
+                f"Submission contains row_id {r['row_id']} which doesn't match the dataset's qid"
+            )
+    print('validated')
+
+def compute_scores_causalvqa(submission_df, split="test"):
+    #assert split == "test", format_error(f"Split {split} not available for dataset {WMQA_NAME}")
+    split = "train"
+    gold_tasks = get_dataset_config_names(WMQA_DATASET, token=TOKEN)
+    subsets = submission_df.data_name.unique()
+    scored_subs = []
+    for subset in subsets:
+        sub_df = submission_df[(submission_df.data_name == subset)].copy()
+        sub_df['model_answer'] = sub_df['model_answer'].str.replace(r'[^a-eA-E]', '', regex=True, flags=re.IGNORECASE).str.upper()
+        gold_dataset = load_dataset(WMQA_DATASET, "", split="train", token=TOKEN)
+        gold_dataset = gold_dataset.to_pandas() 
+        gold_dataset['row_id'] = gold_dataset.apply(lambda x: x['id']+'_'+str(x['n']), axis=1)
+        joined = pd.merge(gold_dataset, sub_df, on='row_id', how='left')
+        correct = []
+        for i, r in joined.iterrows():
+            gold_answer = r['answer']
+            model_answer = r["model_answer"]
+            if gold_answer == model_answer:
+                correct.append(1)
+            else:
+                correct.append(0)
+        joined["rating"] = correct
+        scored_subs.append(joined)
+    print(joined.columns)
+    print('scored')
+    return pd.concat(scored_subs)
+
+def aggregate_scores_causalvqa(scored_submission_df, split="test"):
+    subsets = scored_submission_df.data_name.unique()
+    subset_scores = {f"score_{s}": 0 for s in subsets}
+    for subset in subsets:
+        sub_df = scored_submission_df[scored_submission_df.data_name == subset].copy()
+        agg_df = sub_df.groupby(['id','strata'])['rating'].sum().reset_index()
+        agg_df['points'] = 0
+        agg_df.loc[agg_df['rating']==2, 'points'] = 1
+
+        
+        # compute macro scores
+        subset_scores[f"score_{subset}"] = agg_df.points.mean()*100.00
+    print('aggregated')
+    return subset_scores
+
+
+# --- IntPhys functions ---
+
+
+def validate_intphys(submission_df, split="test"):
+    assert split == "test", format_error(f"Split {split} not available for dataset {INTP_NAME}")
+    subsets = submission_df.data_name.unique()
+    for subset in subsets:
+        assert subset in [INTP_KEY], format_error(
+            f"Wrong tasks, got {subset} but expecting " + INTP_KEY
+        )
+    gold_tasks = get_dataset_config_names(INTP_DATASET, token=TOKEN)
+    for subset in subsets:
+        sub_df = submission_df[(submission_df.data_name == subset)].copy()
+        gold_dataset = load_dataset(INTP_DATASET, "", split="test")
+        assert len(sub_df) == len(gold_dataset), format_error(
+            f"Number of examples do not match in user submission, found {len(sub_df)} but expecting {len(gold_dataset)} in split {subset}"
+        )
+        id2answer = {row["name"]: row["answer"] for row in gold_dataset}
+        for i, r in sub_df.iterrows():
+            assert r["row_id"] in id2answer, format_error(
+                f"Submission contains row_id {r['row_id']} which doesn't match the dataset's video_id"
+            )
+
+
+
+def compute_scores_intphys(submission_df, split="test"):
+    assert split == "test", format_error(f"Split {split} not available for dataset {INTP_NAME}")
+    gold_tasks = get_dataset_config_names(INTP_DATASET, token=TOKEN)
+    subsets = submission_df.data_name.unique()
+    scored_subs = []
+    for subset in subsets:
+        sub_df = submission_df[(submission_df.data_name == subset)].copy()
+        gold_dataset = load_dataset(INTP_DATASET, "", split="test", token=TOKEN)
+        id2answer = {row["name"]: row["answer"] for row in gold_dataset}
+        correct = []
+        for i, r in sub_df.iterrows():
+            gold_answer = id2answer[r["row_id"]]
+            model_answer = r["model_answer"]
+            if gold_answer == model_answer:
+                correct.append(1)
+            else:
+                correct.append(0)
+        sub_df["rating"] = correct
+        scored_subs.append(sub_df)
+    return pd.concat(scored_subs)
+
+
+def aggregate_scores_intphys(scored_submission_df, split="test"):
+    subsets = scored_submission_df.data_name.unique()
+    subset_scores = {f"score_{s}": 0 for s in subsets}
+    accuracies = []
+    for subset in subsets:
+        sub_df = scored_submission_df[
+            (scored_submission_df.data_name == subset)
+        ].copy()
+        result_by_vid = {}
+        pair_correct_count = 0
+        for i, row in sub_df.iterrows():
+            if row["rating"] == 1:
+                pair_correct_count += 1
+        accuracies.append((pair_correct_count / len(sub_df)) * 100)
+        # compute macro scores
+        subset_scores[f"score_{subset}"] = np.mean(accuracies)
+    return subset_scores
+
+
+
+
+VALIDATION_FN = {
+    MVP_KEY: validate_mvp,
+    MVP_MINI_KEY: validate_mvp,
+    INTP_KEY: validate_intphys,
+    WMQA_KEY: validate_causalvqa,
+}
+
+SCORER_FN = {
+    MVP_KEY: compute_scores_mvp,
+    MVP_MINI_KEY: compute_scores_mvp,
+    INTP_KEY: compute_scores_intphys,
+    WMQA_KEY: compute_scores_causalvqa,
+}
+
+AGGREGATE_FN = {
+    MVP_KEY: aggregate_scores_mvp,
+    MVP_MINI_KEY: aggregate_scores_mvp,
+    INTP_KEY: aggregate_scores_intphys,
+    WMQA_KEY: aggregate_scores_causalvqa,
+}
+
+
+def compute_scores(submission_df, split="valid"):
+    """
+    Runs the scores with held out valid/test sets, and updates the submission with metrics for each dataset
+    - First, runs validation for the input to ensure the right keys are present
+    - Then, runs the evaluations
+    """
+    tasks = submission_df.data_name.unique()
+    scored_subs = []
+    for t in tasks:
+        task_sub = submission_df[submission_df.data_name == t].copy()
+        scored_subs.append(SCORER_FN[t](task_sub, split))
+    scored_subs = pd.concat(scored_subs)
+    return scored_subs
+
+
+def aggregate_scores(scored_df, split="valid"):
+    tasks = scored_df.data_name.unique()
+    agg_scores = {}
+    for task in tasks:
+        task_sub = scored_df[scored_df.data_name == task].copy()
+        agg_metrics = AGGREGATE_FN[task](task_sub, split=split)
+        agg_scores.update(agg_metrics)
+    return agg_scores
+
+
+def validate_submission(submission_df, split="valid"):
+    """
+    Validate user submissions
+    """
+    # Run checks
+    assert "data_name" in submission_df.columns, format_error("Submission missing column data_name")
+    assert "row_id" in submission_df.columns, format_error("Submission missing column row_id")
+    assert "task" in submission_df.columns, format_error("Submission missing column task")
+    assert "model_answer" in submission_df.columns, format_error("Submission missing column model_answer")
+    tasks = submission_df.data_name.unique()
+    valid_tasks = [t[0] for t in TASKS] + [MVP_MINI_KEY]
+    for t in tasks:
+        assert t in valid_tasks, format_error(
+            f"Submission contains one or more rows with data_name={t}, which is not a valid task for this leaderboard (expecting to match a dataset in {valid_tasks})"
+        )
+    # Dataset specific checks
+    for task in tasks:
+        task_sub = submission_df[submission_df.data_name == task].copy()
+        VALIDATION_FN[task](task_sub)
+
+
+def add_new_eval(
+    model: str,
+    vision_backbone: str,
+    llm_backbone: str,
+    url: str,
+    model_type: str,
+    path_to_file: str,
+    organization: str,
+    mail: str,
+    profile: gr.OAuthProfile,
+    progress=gr.Progress(),
+):
+    progress(0, desc="Validating user ...")
+    contact_infos = load_dataset(
+        CONTACT_DATASET,
+        token=TOKEN,
+        download_mode="force_redownload",
+        verification_mode=VerificationMode.NO_CHECKS,
+        trust_remote_code=True,
+    )
+    user_submission_dates = sorted(
+        row["date"] for row in contact_infos["test"] if row["username"] == profile.username
+    )
+    # Logic to limit submissions per day
+    if len(user_submission_dates) > 0 and user_submission_dates[-1] == datetime.datetime.today().strftime("%Y-%m-%d"):
+       return format_error("You already submitted once today, please try again tomorrow.")
+    # Very basic email parsing
+    _, parsed_mail = parseaddr(mail)
+    if not "@" in parsed_mail:
+        return format_warning("Please provide a valid email adress.")
+
+    print("Adding new eval")
+    progress(0.1, desc="Fetching recent evals ...")
+
+    eval_results, _, _ = get_eval_data()
+    # # Check if the combination model/org already exists and prints a warning message if yes
+    # if model.lower() in set([m.lower() for m in eval_results[val_or_test]["model"]]) and organization.lower() in set(
+    #     [o.lower() for o in eval_results[val_or_test]["organization"]]
+    # ):
+    #     return format_warning("This model has been already submitted.")
+
+    if path_to_file is None:
+        return format_warning("Please attach a file.")
+
+    # validate submission - do not save submission until its fully validated
+    progress(0.3, desc="Validating user submission ...")
+    file_path = path_to_file.name
+    assert file_path.endswith(".jsonl"), format_error("Please submit a jsonl file")
+    submissions_df = pd.read_json(file_path, lines=True, orient="records")
+    validate_submission(submissions_df)
+
+    # Save submitted file
+    if LOCAL_DEBUG:
+        gr.Info("In local debug mode, mock uploading submission dataset.")
+    else:
+        api.upload_file(
+            repo_id=SUBMISSION_DATASET,
+            path_or_fileobj=path_to_file.name,
+            path_in_repo=f"{organization}/{model}/submissions/test_raw_{datetime.datetime.today()}.jsonl",
+            repo_type="dataset",
+            token=TOKEN,
+        )
+
+    # Compute score
+    progress(0.5, desc="Computing scores ...")
+    scored_df = compute_scores(submissions_df, split="test")
+
+    # Save scored file
+    if LOCAL_DEBUG:
+        gr.Info("In local debug mode, mock uploading scored files")
+    else:
+        tasks = scored_df.data_name.unique()
+        for task in tasks:
+            scored_df.to_json(f"scored/{organization}_{model}_{task}.jsonl", lines=True, orient="records")
+            api.upload_file(
+                repo_id=SUBMISSION_DATASET,
+                path_or_fileobj=f"scored/{organization}_{model}_{task}.jsonl",
+                path_in_repo=f"{organization}/{model}/scored/{task}/test_scored_{datetime.datetime.today()}.jsonl",
+                repo_type="dataset",
+                token=TOKEN,
+            )
+
+    # Actual submission
+    progress(0.7, desc="Submitting leaderboard entry ...")
+    eval_entry = {
+        "model": model,
+        "model_type": model_type,
+        "vision_backbone": vision_backbone,
+        "llm_backbone": llm_backbone,
+        "url": url,
+        "organization": organization,
+        "submitted_by": profile.username,
+        "date": datetime.datetime.today().strftime("%Y-%m-%d"),
+    }
+    agg_metrics = aggregate_scores(scored_df, split="test")
+    eval_entry.update(agg_metrics)
+    # update missing tasks to MISSING_VALUE
+    task_keys = [t[0] for t in TASKS] + [MVP_MINI_KEY]
+    missing_metrics = {f"score_{task}": MISSING_VALUE for task in task_keys if f"score_{task}" not in eval_entry}
+    eval_entry.update(missing_metrics)
+
+    eval_results["test"] = eval_results["test"].add_item(eval_entry)
+    if LOCAL_DEBUG:
+        print(eval_results["valid"][-1])
+        gr.Info("In local debug mode, mock uploading aggregated scores")
+    else:
+        eval_results.push_to_hub(RESULTS_DATASET, token=TOKEN)
+
+    progress(0.9, desc="Updating contacts ...")
+    contact_info = {
+        "model": model,
+        "url": url,
+        "organization": organization,
+        "username": profile.username,
+        "mail": mail,
+        "date": datetime.datetime.today().strftime("%Y-%m-%d"),
+    }
+    contact_infos["test"] = contact_infos["test"].add_item(contact_info)
+    if LOCAL_DEBUG:
+        print("mock uploaded contact info")
+    else:
+        contact_infos.push_to_hub(CONTACT_DATASET, token=TOKEN)
+
+    progress(1.0, desc="Completed evaluation successfully. Please refresh leaderboard")
+    success_str = f"Model {model} submitted by {organization} is successfully evaluated and stored in our database.\nPlease wait a few hours and refresh the leaderboard to see your score displayed."
+    format_log(success_str)
+    return success_str
+
+
+def on_filter_model_size_method_change():
+    _, eval_dataframe_val, eval_dataframe_test = get_eval_data()
+    # eval_dataframe_val = eval_dataframe_val[PRE_COL_NAMES + [f"{t} (%)" for t in selected_columns] + POST_COL_NAMES]
+    eval_dataframe_test = eval_dataframe_test[PRE_COL_NAMES + [f"{t} (%)" for _,t in VISIBLE_TASKS] + POST_COL_NAMES]
+    datatypes = ["markdown"] + ["number" for _ in VISIBLE_TASKS] + ["text"] + ["text"] + ["text"] + ["date"]
+    # val_ldb = gr.components.Dataframe(
+    #     value=eval_dataframe_val, datatype=datatypes, interactive=False, column_widths=["20%"]
+    # )
+    test_ldb = gr.components.Dataframe(
+        value=eval_dataframe_test, datatype=datatypes, interactive=False, column_widths=["20%"]
+    )
+    return test_ldb
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+if __name__ == "__main__":
+
+    _, eval_dataframe_val, eval_dataframe_test = get_eval_data()
+    demo = gr.Blocks()
+    with demo:
+        gr.HTML(TITLE)
+        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+        with gr.Row():
+            with gr.Accordion("📙 Citation", open=False):
+                gr.Markdown(CITATION_BUTTON_LABEL)
+                gr.Markdown(CITATION_BUTTON_TEXT)
+
+
+        datatypes = ["markdown"] + ["number" for _ in VISIBLE_TASKS] + ["text"] + ["text"] + ["text"] + ["date"]
+
+        with gr.Tab("Results: Test"):
+            leaderboard_table_test = gr.components.Dataframe(
+                value=eval_dataframe_test, datatype=datatypes, interactive=False, column_widths=["20%"]
+            )
+
+        refresh_button = gr.Button("Refresh")
+        refresh_button.click(
+        #    print(task_filter)
+            on_filter_model_size_method_change,
+            #inputs=[VISIBLE_TASKS],
+            #inputs=[],
+            outputs=[
+                #leaderboard_table_val,
+                leaderboard_table_test,
+            ],
+        )
+        with gr.Accordion("Submit a new model for evaluation"):
+            with gr.Row():
+                gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    # level_of_test = "test"
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    model_url = gr.Textbox(label="Model URL")
+                    model_type = gr.Dropdown(choices=["Open", "Closed"], label="Model Type")
+                    # num_frames = gr.Textbox(label="Number of frames used")
+                    llm_backbone_textbox = gr.Textbox(label="LLM Backbone")
+                    vision_backbone_textbox = gr.Textbox(label="Vision Backbone")
+                    # system_prompt_textbox = gr.Textbox(label="System prompt example")
+                    # url_textbox = gr.Textbox(label="Url to model information")
+                with gr.Column():
+                    organization = gr.Textbox(label="Organization")
+                    mail = gr.Textbox(
+                        label="Contact email"
+                    )
+                    file_output = gr.File()
+                    submission_result = gr.Textbox(label="Status")
+                    with gr.Row():
+                        with gr.Column():
+                            gr.LoginButton()
+                        with gr.Column():
+                            submit_button = gr.Button("Submit Eval")
+
+            submit_button.click(
+                add_new_eval,
+                [
+                    #level_of_test,
+                    model_name_textbox,
+                    vision_backbone_textbox,
+                    llm_backbone_textbox,
+                    model_url,
+                    model_type,
+                    # num_frames,
+                    file_output,
+                    organization,
+                    mail,
+                ],
+                submission_result,
+            )
+
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(restart_space, "interval", seconds=3600)
+    scheduler.start()
+    demo.launch(debug=True)

content.py

@@ -0,0 +1,90 @@
+import gradio as gr
+
+TITLE = """<h1 align="center" id="space-title">Leaderboard: Physical Reasoning from Video</h1>"""
+
+INTRODUCTION_TEXT = """
+This leaderboard tracks the progress of frontier models on 3 physical reasoning benchmark datasets released by Meta FAIR -- **Minimal Video Pairs (MVPBench)**, **IntPhys 2**, and **CausalVQA**. In addition to tracking the progress of the community through public submissions, we also present human scores for each benchmark to understand the gap between leading models and human performance on key physical and video reasoning tasks. 
+
+- **[MVPBench](https://github.com/facebookresearch/minimal_video_pairs):** A Video Question Answering (VQA) benchmark for spatio-temporal and intuitive physics video understanding. Videos were sourced from diverse datasets and automatically paired together, such that videos in each pair differ in only minimal ways, but have opposing correct answers for the same question. This design ensures that models need to go beyond relying on surface visual or textual biases in order to perform well on the benchmark.  
+
+- **[IntPhys 2](https://github.com/facebookresearch/IntPhys2):** A video benchmark designed to evaluate the intuitive physics understanding of deep learning models. IntPhys 2 focuses on four core principles: Permanence, Immutability, Spatio-Temporal Continuity, and Solidity, and offers a comprehensive suite of tests based on the violation of expectation framework, which challenge models to differentiate between possible and impossible events within controlled and diverse virtual environments. 
+
+- **[CausalVQA](https://github.com/facebookresearch/CausalVQA):** A Video Question Answering (VQA) benchmark composed of question-answer pairs that probe models’ understanding of causality in the physical world. Questions were designed to be grounded in real-world scenarios, while focusing on models’ ability to predict the likely outcomes of different actions and events through five question types – counterfactual, hypothetical, anticipation, planning and descriptive.
+
+Please see the linked GitHub repos for instructions on how to download and run each benchmark, and the instructions below on how to submit results to the leaderboard. 
+"""
+
+SUBMISSION_TEXT = """
+## Submissions
+Scores are calculated according to the metrics defined in each dataset in the leaderboard. Please see each datasets' linked repos below for more details on metric calculation. 
+
+- **[MVPBench](https://github.com/facebookresearch/minimal_video_pairs):** The leaderboard computes and tracks performance on the _"mini"_ split of MVPBench, which contains ~5k items. 
+
+- **[IntPhys 2](https://github.com/facebookresearch/IntPhys2):** The leaderboard computes and tracks performance on the _"held out"_ split of IntPhys 2, a set of 344 simulated videos with moving cameras, realistic objects, and complex backgrounds.
+
+- **[CausalVQA](https://github.com/facebookresearch/CausalVQA):** The leaderboard computes and tracks performance on the _"held out"_ split of CausalVQA, which contains 793 pairs of curated questions.
+
+Note that for IntPhys 2 and CausalVQA, submitting to this leaderboard is the only way to compute accuracy on the _held-out_ splits, as the correct answers on those splits are stored privately and not publicly available.
+
+## Format
+
+Each question calls for an answer that is either a string (one or a few words), a number, or a comma separated list of strings or floats, unless specified otherwise. There is only one correct answer. 
+
+
+We expect submissions to be json-line files with the following format:
+```
+{"data_name": "mvp", "task": "some_task", "row_id": 0, "model_answer": "Answer 1 from your model"}
+{"data_name": "mvp", "task": "some_task", "row_id" : 1, "model_answer": "Answer 2 from your model"}
+```
+
+Here, `data_name` can be one of three valid options: `mvp_mini`, `intphys2`, `causalvqa`. You can merge your results from multiple datasets into a single submission. 
+If the dataset contains multiple tasks (as in case of MVPBench and CausalVQA), the `task` field is used to identify the subtask. For datasets that do not have `task`, this field can be kept blank; however, the submission file should have this field in the json.
+For each submission we receive, we will check if all rows from the dataset are valid.
+
+"""
+
+CITATION_BUTTON_LABEL = "Please cite the following papers if you use these benchmarks"
+CITATION_BUTTON_TEXT = r"""
+```latex
+@misc{mvpbench,
+      title={A Shortcut-aware Video-QA Benchmark for Physical Understanding via Minimal Video Pairs}, 
+      author={Benno Krojer and Mojtaba Komeili and Candace Ross and Quentin Garrido and Koustuv Sinha and Nicolas Ballas and Mahmoud Assran},
+      year={2025},
+      eprint={},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{causalvqa,
+      title={CausalVQA: A Physically Grounded Causal Reasoning Benchmark for Video Models}, 
+      author={Aaron Foss and Chloe Evans and Sasha Mitts and Koustuv Sinha and Ammar Rizvi and Justine T Kao},
+      year={2025},
+      eprint={},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{intphys2,
+      title={IntPhys 2: Benchmarking Intuitive Physics Understanding In Complex Synthetic Environments}, 
+      author={Florian Bordes and Quentin Garrido and Justine T Kao and Adina Williams and Michael Rabbat and Emmanuel Dupoux},
+      year={2025},
+      eprint={},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+"""
+
+
+def format_error(msg):
+    raise gr.Error(msg)
+
+
+def format_warning(msg):
+    return gr.Warning(msg)
+
+
+def format_log(msg):
+    return gr.Info(msg)
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

scorer.py

@@ -0,0 +1,104 @@
+import json
+import re
+import string
+import warnings
+
+import numpy as np
+
+
+def normalize_number_str(number_str: str) -> float:
+    # we replace these common units and commas to allow
+    # conversion to float
+    for char in ["$", "%", ","]:
+        number_str = number_str.replace(char, "")
+    try:
+        return float(number_str)
+    except ValueError:
+        print(f"String {number_str} cannot be normalized to number str.")
+        return float("inf")
+
+
+def split_string(
+    s: str,
+    char_list: list[str] = [",", ";"],
+) -> list[str]:
+    pattern = f"[{''.join(char_list)}]"
+    return re.split(pattern, s)
+
+
+def question_scorer(
+    model_answer: str,
+    ground_truth: str,
+) -> bool:
+    def is_float(element: any) -> bool:
+        try:
+            float(element)
+            return True
+        except ValueError:
+            return False
+        
+    if model_answer is None:
+        model_answer = "None"
+
+    # if gt is a number
+    if is_float(ground_truth):
+        print(f"Evaluating {model_answer} as a number.")
+        normalized_answer = normalize_number_str(model_answer)
+        return normalized_answer == float(ground_truth)
+
+    # if gt is a list
+    elif any(char in ground_truth for char in [",", ";"]):
+        print(f"Evaluating {model_answer} as a comma separated list.")
+        # question with the fish: normalization removes punct
+
+        gt_elems = split_string(ground_truth)
+        ma_elems = split_string(model_answer)
+
+        # check length is the same
+        if len(gt_elems) != len(ma_elems):
+            warnings.warn(
+                "Answer lists have different lengths, returning False.", UserWarning
+            )
+            return False
+
+        # compare each element as float or str
+        comparisons = []
+        for ma_elem, gt_elem in zip(ma_elems, gt_elems):
+            if is_float(gt_elem):
+                normalized_ma_elem = normalize_number_str(ma_elem)
+                comparisons.append(normalized_ma_elem == float(gt_elem))
+            else:
+                # we do not remove punct since comparisons can include punct
+                comparisons.append(
+                    normalize_str(ma_elem, remove_punct=False)
+                    == normalize_str(gt_elem, remove_punct=False)
+                )
+        return all(comparisons)
+
+    # if gt is a str
+    else:
+        print(f"Evaluating {model_answer} as a string.")
+        return normalize_str(model_answer) == normalize_str(ground_truth)
+
+
+def normalize_str(input_str, remove_punct=True) -> str:
+    """
+    Normalize a string by:
+    - Removing all white spaces
+    - Optionally removing punctuation (if remove_punct is True)
+    - Converting to lowercase
+    Parameters:
+    - input_str: str, the string to normalize
+    - remove_punct: bool, whether to remove punctuation (default: True)
+    Returns:
+    - str, the normalized string
+    """
+    # Remove all white spaces. Required e.g for seagull vs. sea gull
+    no_spaces = re.sub(r"\s", "", input_str)
+
+    # Remove punctuation, if specified.
+    if remove_punct:
+        translator = str.maketrans("", "", string.punctuation)
+        return no_spaces.lower().translate(translator)
+    else:
+        return no_spaces.lower()

utils.py

@@ -0,0 +1,15 @@
+import re
+
+def extract_single_option(video_llm_output) -> Union[str, bool]:
+    video_llm_output = video_llm_output.lower()
+    pattern = r"(Answer|Assistant)?:?\s*([AB])\b"
+    matches = re.findall(pattern, video_llm_output, re.IGNORECASE)
+    if matches:
+        actual_answer = matches[1] if len(matches) > 1 else matches[0]
+        answer = actual_answer[1].lower()
+        return answer 
+    elif len(video_llm_output) == 1:
+        answer = video_llm_output.lower()
+        return answer 
+    else:
+        return False