feat: update leaderboard with .json from HF

app.py

@@ -3,13 +3,87 @@ import os
 
 import gradio as gr
 import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub import HfApi, hf_hub_download, get_collection
 from huggingface_hub.repocard import metadata_load
+from typing import Dict
+
+
+def get_datasets_nickname() -> Dict: 
+
+    datasets_nickname = {}
+
+    collection = get_collection("vidore/vidore-benchmark-667173f98e70a1c0fa4db00d")
+
+    collection_items = collection.items
+
+    for item in collection_items:
+        dataset_name = item.item_id
+
+        if 'arxivqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'ArxivQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'ArxivQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'ArxivQA'
+        
+        elif 'docvqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'DocVQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'DocVQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'DocVQA'
+
+        elif 'infovqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'InfoVQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'InfoVQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'InfoVQA'
+        
+        elif 'tabfquad' in dataset_name:
+            datasets_nickname[dataset_name] = 'TabFQuad'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'TabFQuad'
+            datasets_nickname[dataset_name + '_captioning'] = 'TabFQuad'
+        
+        elif 'tatdqa' in dataset_name:
+            datasets_nickname[dataset_name] = 'TATDQA'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'TATDQA'
+            datasets_nickname[dataset_name + '_captioning'] = 'TATDQA'
+        
+        elif 'shiftproject' in dataset_name:
+            datasets_nickname[dataset_name] = 'ShiftProject'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'ShiftProject'
+            datasets_nickname[dataset_name + '_captioning'] = 'ShiftProject'
+        
+        elif 'artificial_intelligence' in dataset_name:
+            datasets_nickname[dataset_name] = 'Artificial Intelligence'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Artificial Intelligence'
+            datasets_nickname[dataset_name + '_captioning'] = 'Artificial Intelligence'
+        
+        elif 'energy' in dataset_name:
+            datasets_nickname[dataset_name] = 'Energy'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Energy'
+            datasets_nickname[dataset_name + '_captioning'] = 'Energy'
+        
+        elif 'government_reports' in dataset_name:
+            datasets_nickname[dataset_name] = 'Government Reports'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Government Reports'
+            datasets_nickname[dataset_name + '_captioning'] = 'Government Reports'
+
+        elif 'healthcare' in dataset_name:
+            datasets_nickname[dataset_name] = 'Healthcare'
+            datasets_nickname[dataset_name + '_ocr_chunk'] = 'Healthcare'
+            datasets_nickname[dataset_name + '_captioning'] = 'Healthcare'
+
+    return datasets_nickname
 
 
 def make_clickable_model(model_name, link=None):
+    
     if link is None:
-        link = "https://huggingface.co/" + model_name
+        desanitized_model_name = model_name.replace("_", "/")
+        
+        if '/captioning' in desanitized_model_name:
+            desanitized_model_name = desanitized_model_name.replace('/captioning', '')
+        if '/ocr' in desanitized_model_name:
+            desanitized_model_name = desanitized_model_name.replace('/ocr', '')
+
+        link = "https://huggingface.co/" + desanitized_model_name
+        
     # Remove user from model name
     # return (
     #     f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name.split("/")[-1]}</a>'
@@ -47,40 +121,53 @@ def get_vidore_data():
 
     # local cache path
     model_infos_path = "model_infos.json"
+    metric = "ndcg_at_5"
+
+
     MODEL_INFOS = {}
     if os.path.exists(model_infos_path):
         with open(model_infos_path) as f:
             MODEL_INFOS = json.load(f)
 
     models = api.list_models(filter="vidore")
-
-    for model in models:
-        if model.modelId not in MODEL_INFOS:
-            readme_path = hf_hub_download(model.modelId, filename="README.md")
-            meta = metadata_load(readme_path)
-            try:
-                result_path = hf_hub_download(model.modelId, filename="results.json")
-
-                with open(result_path) as f:
-                    results = json.load(f)
-                # keep only ndcg_at_5
-                for dataset in results:
-                    results[dataset] = {key: value for key, value in results[dataset].items() if "ndcg_at_5" in key}
-
-                MODEL_INFOS[model.modelId] = {"metadata": meta, "results": results}
-            except:
-                continue
+    repositories = [model.modelId for model in models]  
+
+    datasets_nickname = get_datasets_nickname()
+    for repo_id in repositories:
+        files = [f for f in api.list_repo_files(repo_id) if f.endswith('_metrics.json')]
+        if len(files) == 0:
+            continue
+        else :
+            for file in files:
+                model_name = file.split('_metrics.json')[0]
+
+                if model_name not in MODEL_INFOS:
+                    readme_path = hf_hub_download(repo_id, filename="README.md")
+                    meta = metadata_load(readme_path)
+                    try:
+                        result_path = hf_hub_download(repo_id, filename= file)
+
+                        with open(result_path) as f:
+                            results = json.load(f)
+                        # keep only ndcg_at_5
+                        for dataset in results:
+                            results[dataset] = {key: value for key, value in results[dataset].items() if metric in key}
+
+                        MODEL_INFOS[model_name] = {"meta":meta, "results": results}
+                    except:
+                        continue
 
     model_res = {}
     df = None
     if len(MODEL_INFOS) > 0:
         for model in MODEL_INFOS.keys():
+            print(model)
             res = MODEL_INFOS[model]["results"]
             dataset_res = {}
             for dataset in res.keys():
                 if "validation_set" == dataset:
                     continue
-                dataset_res[dataset] = res[dataset]["ndcg_at_5"]
+                dataset_res[datasets_nickname[dataset]] = res[dataset][metric]
             model_res[model] = dataset_res
 
         df = pd.DataFrame(model_res).T