refactor : break app.py in different files

.gitignore

@@ -0,0 +1,3 @@
+.venv
+*.json
+*.pyc

app/__init__.py

@@ -0,0 +1 @@
+

app/utils.py

@@ -0,0 +1,31 @@
+from data.model_handler import ModelHandler
+
+def make_clickable_model(model_name, link=None):
+    if link is None:
+        desanitized_model_name = model_name.replace("_", "/")
+
+        if '/captioning' in desanitized_model_name:
+            desanitized_model_name = desanitized_model_name.replace('/captioning', '')
+        if '/ocr' in desanitized_model_name:
+            desanitized_model_name = desanitized_model_name.replace('/ocr', '')
+
+        link = "https://huggingface.co/" + desanitized_model_name
+
+    return f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name}</a>'
+
+
+def add_rank_and_format(df):
+    df = df.reset_index()
+    df = df.rename(columns={"index": "Model"})
+    df = ModelHandler.add_rank(df)
+    df["Model"] = df["Model"].apply(make_clickable_model)
+    return df
+
+def get_refresh_function():
+    def _refresh(metric):
+        model_handler = ModelHandler()
+        data_task_category = model_handler.get_vidore_data(metric)
+        df = add_rank_and_format(data_task_category)
+        return df
+
+    return _refresh

data/__init__.py

@@ -0,0 +1 @@
+

data/dataset_handler.py

@@ -0,0 +1,64 @@
+from typing import Dict
+from huggingface_hub import get_collection
+
+
+def get_datasets_nickname() -> Dict:
+    datasets_nickname = {}
+
+    collection = get_collection("vidore/vidore-benchmark-667173f98e70a1c0fa4db00d")
+    collection_items = collection.items
+
+    for item in collection_items:
+        dataset_name = item.item_id
+
+        if 'arxivqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'ArxivQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'ArxivQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'ArxivQA'
+
+        elif 'docvqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'DocVQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'DocVQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'DocVQA'
+
+        elif 'infovqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'InfoVQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'InfoVQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'InfoVQA'
+
+        elif 'tabfquad' in dataset_name:
+            datasets_nickname[dataset_name] = 'TabFQuad'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'TabFQuad'
+            datasets_nickname[dataset_name + '_captioning'] = 'TabFQuad'
+
+        elif 'tatdqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'TATDQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'TATDQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'TATDQA'
+
+        elif 'shiftproject' in dataset_name:
+            datasets_nickname[dataset_name] = 'ShiftProject'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'ShiftProject'
+            datasets_nickname[dataset_name + '_captioning'] = 'ShiftProject'
+
+        elif 'artificial_intelligence' in dataset_name:
+            datasets_nickname[dataset_name] = 'Artificial Intelligence'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Artificial Intelligence'
+            datasets_nickname[dataset_name + '_captioning'] = 'Artificial Intelligence'
+
+        elif 'energy' in dataset_name:
+            datasets_nickname[dataset_name] = 'Energy'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Energy'
+            datasets_nickname[dataset_name + '_captioning'] = 'Energy'
+
+        elif 'government_reports' in dataset_name:
+            datasets_nickname[dataset_name] = 'Government Reports'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Government Reports'
+            datasets_nickname[dataset_name + '_captioning'] = 'Government Reports'
+
+        elif 'healthcare' in dataset_name:
+            datasets_nickname[dataset_name] = 'Healthcare'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Healthcare'
+            datasets_nickname[dataset_name + '_captioning'] = 'Healthcare'
+
+    return datasets_nickname

data/model_handler.py

@@ -0,0 +1,94 @@
+import json
+import os
+from typing import Dict
+from huggingface_hub import HfApi, hf_hub_download, metadata_load
+import pandas as pd
+from .dataset_handler import get_datasets_nickname
+
+class ModelHandler:
+    def __init__(self, model_infos_path="model_infos.json"):
+        self.api = HfApi()
+        self.model_infos_path = model_infos_path
+        self.model_infos = self._load_model_infos()
+
+    def _load_model_infos(self) -> Dict:
+        if os.path.exists(self.model_infos_path):
+            with open(self.model_infos_path) as f:
+                return json.load(f)
+        return {}
+
+    def _save_model_infos(self):
+        with open(self.model_infos_path, "w") as f:
+            json.dump(self.model_infos, f)
+
+    def get_vidore_data(self, metric="ndcg_at_5"):
+        models = self.api.list_models(filter="vidore")
+        repositories = [model.modelId for model in models]  # type: ignore
+
+        datasets_nickname = get_datasets_nickname()
+        for repo_id in repositories:
+            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith('_metrics.json')]
+            if len(files) == 0:
+                continue
+            else:
+                for file in files:
+                    model_name = file.split('_metrics.json')[0]
+
+                    if model_name not in self.model_infos:
+                        readme_path = hf_hub_download(repo_id, filename="README.md")
+                        meta = metadata_load(readme_path)
+                        try:
+                            result_path = hf_hub_download(repo_id, filename=file)
+
+                            with open(result_path) as f:
+                                results = json.load(f)
+
+                            for dataset in results:
+                                results[dataset] = {key: value for key, value in results[dataset].items()}
+
+                            self.model_infos[model_name] = {"meta": meta, "results": results}
+                        except Exception as e:
+                            print(f"Error loading {model_name} - {e}")
+                            continue
+
+        #self._save_model_infos()
+
+        model_res = {}
+        if len(self.model_infos) > 0:
+            for model in self.model_infos.keys():
+                res = self.model_infos[model]["results"]
+                dataset_res = {}
+                for dataset in res.keys():
+                    if "validation_set" == dataset:
+                        continue
+                    dataset_res[datasets_nickname[dataset]] = res[dataset][metric]
+                model_res[model] = dataset_res
+                
+            df = pd.DataFrame(model_res).T
+            return df
+        return pd.DataFrame()
+
+    @staticmethod
+    def add_rank(df):
+        cols_to_rank = [
+            col
+            for col in df.columns
+            if col
+            not in [
+                "Model",
+                "Model Size (Million Parameters)",
+                "Memory Usage (GB, fp32)",
+                "Embedding Dimensions",
+                "Max Tokens",
+            ]
+        ]
+        if len(cols_to_rank) == 1:
+            df.sort_values(cols_to_rank[0], ascending=False, inplace=True)
+        else:
+            df.insert(len(df.columns) - len(cols_to_rank), "Average", df[cols_to_rank].mean(axis=1, skipna=False))
+            df.sort_values("Average", ascending=False, inplace=True)
+        df.insert(0, "Rank", list(range(1, len(df) + 1)))
+        df = df.round(2)
+        # Fill NaN after averaging
+        df.fillna("", inplace=True)
+        return df