import gradio as gr from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from huggingface_hub import snapshot_download from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION from src.display.css_html_js import custom_css from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN from src.populate import get_evaluation_queue_df, get_leaderboard_df from src.submission.submit import add_new_eval import random import matplotlib.pyplot as plt import re import plotly.express as px import plotly.graph_objects as go import numpy as np def mean_of_max_per_field(df): """ Calcola il massimo per ciascun campo e poi la media dei massimi. Args: df (pd.DataFrame): DataFrame con colonne TE, SA, HS, AT, WIC, FAQ, LS, SU, NER, REL Returns: float: media dei valori massimi dei campi """ fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] #print(df.columns) # Controlla che tutte le colonne esistano nel DataFrame missing = [f for f in fields if f not in df.columns] if missing: raise ValueError(f"Le seguenti colonne mancano nel DataFrame: {missing}") # Calcola il massimo per ciascun campo max_values = df[fields].max() # Calcola la media dei massimi mean_max = max_values.mean() return mean_max def barplot_mean_few_minus_zero_shot(dataframe, tasks=None): if tasks is None: tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] task_means = {} for task in tasks: if task not in dataframe.columns: continue # Separa few-shot e zero-shot few_shot = dataframe[dataframe['IS_FS'] == True][["Model", task]] zero_shot = dataframe[dataframe['IS_FS'] == False][["Model", task]] # Allinea i modelli merged = pd.merge(few_shot, zero_shot, on="Model", suffixes=("_few", "_zero")) # Rimuovi righe con valori mancanti merged = merged.dropna(subset=[f"{task}_few", f"{task}_zero"]) if merged.empty: continue # Calcola differenza few - zero diff = merged[f"{task}_few"] - merged[f"{task}_zero"] # Calcola la media task_means[task] = diff.mean() # Crea barplot fig = go.Figure([go.Bar( x=list(task_means.keys()), y=list(task_means.values()), marker_color="#ff7f0e", text=[f"{v:.2f}" for v in task_means.values()], textposition="outside", hovertemplate="%{x}
Mean Delta Accuracy: %{y:.2f}%" )]) # Linea di riferimento a 0 ''' fig.add_shape( type="line", x0=-0.5, x1=len(task_means) - 0.5, y0=0, y1=0, line=dict(color="black", width=2, dash="dash"), xref="x", yref="y" ) ''' fig.update_layout( title="Mean Accuracy Difference (Few-shot − Zero-shot) per Task", xaxis_title="", yaxis_title="Mean Delta Combined Performance", template="plotly_white", font=dict(family="Arial", size=13), #margin=dict(b=100) ) fig.add_annotation( text="5-shot learning generally outperforms zero-shot, especially in tasks like NER and REL.
" "Only in Summarization (SU) does it provide no accuracy gain.", xref="paper", yref="paper", x=0, y=-0.2, showarrow=False, font=dict(size=11, color="gray"), align="left" ) return fig def boxplot_per_task(dataframe=None, baselines=None, references=None): #print(dataframe.columns) tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] if dataframe is None: np.random.seed(42) dataframe = pd.DataFrame({ task: np.random.uniform(0.4, 0.9, 20) * 100 for task in tasks }) if baselines is None: baselines = {task: np.random.randint(50, 70) for task in tasks} colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"] fig = go.Figure() for i, task in enumerate(tasks): if task in dataframe.columns: y_data = dataframe[task].dropna().tolist() # boxplot fig.add_trace(go.Box( y=y_data, name=task, marker=dict(color=colors[i]), line=dict(color="black", width=2), fillcolor=colors[i], opacity=0.7, hovertemplate=""+task+"
Accuracy: %{y:.2f}%", width=0.6, whiskerwidth=0.2, quartilemethod="linear" )) # baseline if task in baselines and baselines[task] is not None: fig.add_shape( type="line", x0=i - 0.3, x1=i + 0.3, y0=baselines[task], y1=baselines[task], line=dict(color="black", width=2, dash="dot"), # più visibile xref="x", yref="y" ) ''' fig.add_annotation( x=i, y=baselines[task], text=f"{baselines[task]}%", showarrow=False, yshift=10, font=dict(size=10, color="black") ) ''' # reference GPT-4o if task in references and references[task] is not None: fig.add_shape( type="line", x0=i - 0.3, x1=i + 0.3, y0=references[task], y1=references[task], line=dict(color="red", width=2, dash="dashdot"), xref="x", yref="y" ) fig.update_layout( title="Distribution of Model Accuracy by Task", xaxis_title="Task", yaxis_title="Combined Performance", template="plotly_white", boxmode="group", dragmode=False, font=dict(family="Arial", size=10), margin=dict(b=80), ) fig.add_annotation( text=( "In tasks like TE and SA, models approach the accuracy of supervised
" "models at EVALITA (dashed black line); in NER and REL they remain lower.
" "Dashed red lines show GPT-4o reference results for generative tasks." ), xref="paper", yref="paper", x=0.5, y=-0.30, showarrow=False, font=dict(size=11, color="gray"), align="left" ) fig.update_yaxes(range=[0, 100], fixedrange=True) return fig # EVALITA results BASELINES = { "TE":71.00, "SA": 66.38, "HS": 80.88, "AT": 82.40, "WIC": 85.00, "LS": 38.82, "SU": 38.91, "NER":88.00, "REL": 62.99 } # GPT-4o REFERENCES = { "NER": 79.11, "REL": 63.32, "LS": 59.25, "SU": 33.04 } def boxplot_prompts_per_task(dataframe, tasks=None): if tasks is None: tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] # Lista delle colonne da aggiornare cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"] # Applichiamo la trasformazione for col in cols_to_update: dataframe[col] = dataframe[col].replace({1: 7, 2: 8}) fig = go.Figure() # Liste per creare una sola voce in legenda per Average e Best avg_x, avg_y = [], [] best_x, best_y, best_text = [], [], [] for task in tasks: avg_col = f"{task} Prompt Average" best_col = f"{task} Best Prompt" best_id_col = f"{task} Best Prompt Id" if all(col in dataframe.columns for col in [avg_col, best_col, best_id_col]): avg_value = dataframe[avg_col].mean() avg_x.append(task) avg_y.append(avg_value) best_value = dataframe[best_col].mean() best_x.append(task) best_y.append(best_value) best_id = dataframe[best_id_col].mode()[0] # Most frequent best prompt id best_text.append(f"P:{best_id}") # Barre Average Accuracy (azzurro) fig.add_trace(go.Bar( x=avg_x, y=avg_y, name="Avg. Accuracy", marker_color="#1f77b4", #hovertemplate="%{y:.2f}%" #hovertemplate="" + task + "
Accuracy: %{y:.2f}%", )) # Barre Best Prompt (rosso) fig.add_trace(go.Bar( x=best_x, y=best_y, name="Best Prompt", marker_color="#d62728", #hovertemplate="%{y:.2f}%" #hovertemplate = "" + task + "
Accuracy: %{y:.2f}%", )) # Testo sopra barre Best Prompt con ID for x, y, text in zip(best_x, best_y, best_text): fig.add_annotation( x=x, y=y + 3, # leggermente sopra la barra text=text, showarrow=False, font=dict(size=12, color="black") ) fig.update_layout( title= "Prompt Accuracy: Avg vs Best", xaxis_title="Task", yaxis_title="Combined Performance", barmode='group', template="plotly_white", font=dict(family="Arial", size=10), yaxis=dict(range=[0, 100], fixedrange=True) ) # caption come annotazione separata fig.add_annotation( text="There is no single prompt that performs best across all tasks.
" "Different prompts achieve the highest accuracy on different tasks.", xref="paper", yref="paper", x=0.5, y=-0.3, showarrow=False, font=dict(size=11, color="gray"), align="center", xanchor="center" ) return fig def line_chart(dataframe): # Normalizza le dimensioni per avere marker non troppo piccoli né enormi def scale_sizes(values, min_size=8, max_size=30): vmin, vmax = min(values), max(values) return [ min_size + (val - vmin) / (vmax - vmin) * (max_size - min_size) if vmax > vmin else (min_size + max_size) / 2 for val in values ] # dati in base a IS_FS df_true = dataframe[dataframe['IS_FS'] == True] df_false = dataframe[dataframe['IS_FS'] == False] # Estrai valori x, y e labels x_true = df_true['#Params (B)'].tolist() y_true = df_true['Avg. Comb. Perf. ⬆️'].tolist() labels_true = [re.search(r'>([^<]+)<', m).group(1) for m in df_true['Model'].tolist()] x_false = df_false['#Params (B)'].tolist() y_false = df_false['Avg. Comb. Perf. ⬆️'].tolist() labels_false = [re.search(r'>([^<]+)<', m).group(1) for m in df_false['Model'].tolist()] fig = go.Figure() # Punti IS_FS=True fig.add_trace(go.Scatter( x=x_true, y=y_true, mode='markers', name='5-Shot', marker=dict( color='blue', size=scale_sizes(x_true) ), hovertemplate='%{customdata}
#Params: %{x}
Performance: %{y}', customdata=labels_true )) # Punti IS_FS=False fig.add_trace(go.Scatter( x=x_false, y=y_false, mode='markers', name='0-Shot', marker=dict( color='red', size=scale_sizes(x_false) ), hovertemplate='%{customdata}
#Params: %{x}
Performance: %{y}', customdata=labels_false )) # Trova il massimo tra tutti i modelli all_y = y_true + y_false all_x = x_true + x_false all_labels = labels_true + labels_false max_idx = all_y.index(max(all_y)) max_x = all_x[max_idx] max_y = all_y[max_idx] max_label = all_labels[max_idx] # Aggiungi annotazione visibile per il modello migliore fig.add_annotation( x=max_x, y=max_y, #text=f"Top: {max_label} ({max_y:.1f}%)", text=f"{max_label}", showarrow=True, arrowhead=2, arrowsize=1, arrowwidth=2, arrowcolor="black", font=dict(size=11, color="black"), xshift=10, yshift=10, ax = -30, ay = -20, # sposta la label a sinistra e sopra il punto xanchor = "right" # allinea la label a destra rispetto al punto ) fig.update_layout( title="Avg. Combined Performance vs #Params", xaxis_title="#Params (B)", yaxis_title="Avg. Combined Performance", template="plotly_white", hovermode="closest", font=dict(family="Arial", size=10), dragmode=False, xaxis=dict( tickvals=[0, 25, 50, 75, 100, 125], ticktext=["0", "25", "50", "75", "100"] ), yaxis=dict( tickvals=[0, 20, 40, 60, 80, 100], # 👈 tick fissi range=[0, 100] # 👈 range bloccato ) ) # Caption fig.add_annotation( text="Accuracy generally rises with #Params, but smaller models
" "with 5-shot can outperform larger zero-shot models.", xref="paper", yref="paper", x=0.5, y=-0.3, # 👈 centrata showarrow=False, font=dict(size=11, color="gray"), align="center", xanchor="center" # 👈 ancora centrata rispetto al testo ) fig.update_xaxes(fixedrange=True, rangeslider_visible=False) fig.update_yaxes(fixedrange=True) return fig # Define task metadata (icons, names, descriptions) TASK_METADATA_MULTIPLECHOICE = { "TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": ""}, "SA": {"icon": "😃", "name": "Sentiment Analysis", "tooltip": ""}, "HS": {"icon": "⚠️", "name": "Hate Speech", "tooltip": ""}, "AT": {"icon": "🏥", "name": "Admission Test", "tooltip": ""}, "WIC": {"icon": "🔤", "name": "Word in Context", "tooltip": ""}, "FAQ": {"icon": "❓", "name": "Frequently Asked Questions", "tooltip": ""} } # Define task metadata (icons, names, descriptions) TASK_METADATA_GENERATIVE = { "LS": {"icon": "🔄", "name": "Lexical Substitution", "tooltip": ""}, "SU": {"icon": "📝", "name": "Summarization", "tooltip": ""}, "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": ""}, "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": ""}, } def restart_space(): """Restart the Hugging Face space.""" API.restart_space(repo_id=REPO_ID) def init_leaderboard(dataframe, default_selection=None, hidden_columns=None): """ Initialize and return the leaderboard when it is first loaded or when 'benchmark' is selected. The table is sorted based on the "Avg. Combined Performance" field. """ if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") #print("????????????????????????????????", mean_of_max_per_field(dataframe)) sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False) sorted_dataframe = sorted_dataframe.reset_index(drop=True) sorted_dataframe["Rank"] = sorted_dataframe.index + 1 # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo large_medal_fs_assigned = False medium_medal_fs_assigned = False small_medal_fs_assigned = False large_medal_0shot_assigned = False medium_medal_0shot_assigned = False small_medal_0shot_assigned = False # Lista temporanea per salvare i nuovi valori della colonna Model new_model_column = [] for _, row in sorted_dataframe.iterrows(): if row['IS_FS']: # 5-Few-Shot if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆") large_medal_fs_assigned = True elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🏆") medium_medal_fs_assigned = True elif row["Size"] == "🔵" and not small_medal_fs_assigned: new_model_column.append(f"{row['Model']} 🔵🏆") small_medal_fs_assigned = True else: new_model_column.append(row["Model"]) else: # 0-Shot if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️") large_medal_0shot_assigned = True elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🎖️") medium_medal_0shot_assigned = True elif row["Size"] == "🔵" and not small_medal_0shot_assigned: new_model_column.append(f"{row['Model']} 🔵🎖️") small_medal_0shot_assigned = True else: new_model_column.append(row["Model"]) # Lista delle colonne da aggiornare #cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"] # Applichiamo la trasformazione #for col in cols_to_update: # dataframe[col] = dataframe[col].replace({1: 7, 2: 8}) # Aggiorna la colonna Model sorted_dataframe["Model"] = new_model_column field_list = fields(AutoEvalColumn) return Leaderboard( value=sorted_dataframe, datatype=[c.type for c in field_list], #select_columns=SelectColumns( # default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default], # cant_deselect=[c.name for c in field_list if c.never_hidden], # label="Select Columns to Display:", #), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=hidden_columns or [c.name for c in field_list if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"), #ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)", # default=[["0️⃣", "0️⃣"]]), ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max = 100, default = [0,100], label="Select the number of parameters (B)"), ], #filter_columns=[ # ColumnFilter("IS_FS", type="checkbox", default=False, label="5-Few-Shot") # #ColumnFilter("FS", type="dropdown", label="5-Few-Shot") #], bool_checkboxgroup_label="Evaluation Mode", interactive=False, ) def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=None): """ Update and return the leaderboard when a specific task is selected. The table is sorted based on the "Combined Performance" field. """ if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") sorted_dataframe = dataframe.sort_values(by="Combined Performance", ascending=False) # aggiungo la colonna rank in base alla posizione sorted_dataframe = sorted_dataframe.reset_index(drop=True) sorted_dataframe["Rank"] = sorted_dataframe.index + 1 # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo large_medal_fs_assigned = False medium_medal_fs_assigned = False small_medal_fs_assigned = False large_medal_0shot_assigned = False medium_medal_0shot_assigned = False small_medal_0shot_assigned = False # Lista temporanea per salvare i nuovi valori della colonna Model new_model_column = [] for _, row in sorted_dataframe.iterrows(): if row['IS_FS']: # 5-Few-Shot if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆") large_medal_fs_assigned = True elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🏆") medium_medal_fs_assigned = True elif row["Size"] == "🔵" and not small_medal_fs_assigned: new_model_column.append(f"{row['Model']} 🔵🏆") small_medal_fs_assigned = True else: new_model_column.append(row["Model"]) else: # 0-Shot if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️") large_medal_0shot_assigned = True elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned: new_model_column.append(f"{row['Model']} 🔵🔵🎖️") medium_medal_0shot_assigned = True elif row["Size"] == "🔵" and not small_medal_0shot_assigned: new_model_column.append(f"{row['Model']} 🔵🎖️") small_medal_0shot_assigned = True else: new_model_column.append(row["Model"]) # Aggiorna la colonna Model sorted_dataframe["Model"] = new_model_column pd.set_option('display.max_colwidth', None) #print("========================", dataframe['Model']) #print(sorted_dataframe['Combined Performance']) field_list = fields(AutoEvalColumn) return Leaderboard( value=sorted_dataframe, #datatype=[c.type for c in field_list], datatype=[c.type for c in field_list] + [int], #select_columns=SelectColumns( # default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default], # cant_deselect=[c.name for c in field_list if c.never_hidden], # label="Select Columns to Display:", #), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=hidden_columns or [c.name for c in field_list if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"), ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100], label="Select the number of parameters (B)"), ], bool_checkboxgroup_label="Evaluation Mode", interactive=False ) ''' # Helper function for leaderboard initialization def init_leaderboard(dataframe, default_selection=None, hidden_columns=None): """Initialize and return a leaderboard.""" if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"), ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=150, label="Select the number of parameters (B)"), ], bool_checkboxgroup_label="Hide models", interactive=False, ) ''' def download_snapshot(repo, local_dir): """Try to download a snapshot from Hugging Face Hub.""" try: print(f"Downloading from {repo} to {local_dir}...") snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN) except Exception as e: print(f"Error downloading {repo}: {e}") restart_space() # Initialize the app by downloading snapshots download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH) download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH) # Load leaderboard data LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) #print(LEADERBOARD_DF.columns.tolist()) theoretical_max_combined_perf = mean_of_max_per_field(LEADERBOARD_DF) # Prepare the main interface demo = gr.Blocks(css=custom_css) with demo: #gr.HTML(TITLE) gr.HTML( """

EVALITA-LLM Leaderboard

Open Italian LLM Leaderboard
""" ) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") # ⬇️ QUI aggiungiamo i grafici subito sotto la barra del titolo e sopra le tabs with gr.Row(): gr.Plot(value=line_chart(LEADERBOARD_DF), elem_id="line-chart") gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES, REFERENCES), elem_id="boxplot-task") #gr.Plot(value=boxplot_prompts_per_task(LEADERBOARD_DF), elem_id="boxplot-prompt-task") with gr.Tabs(elem_classes="tab-buttons") as tabs: # Main leaderboard tab with gr.TabItem("🏅 Benchmark"): leaderboard = init_leaderboard( LEADERBOARD_DF, default_selection=['Rank', 'Size', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"], hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]] ) gr.HTML( f"""
Theoretical performance of a model that scores the highest on every individual task: {theoretical_max_combined_perf:.2f}
""" ) ''' with gr.TabItem("📈 Charts"): #gr.Plot(value=line_chart(LEADERBOARD_DF), label="Andamento di esempio") #gr.Plot(value=line_chart_interactive_test(), label="Andamento interattivo") gr.Plot(value=line_chart(LEADERBOARD_DF)) gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES)) gr.Plot(value=boxplot_prompts_per_task(LEADERBOARD_DF)) gr.Plot(value=barplot_mean_few_minus_zero_shot(LEADERBOARD_DF)) ''' # About tab with gr.TabItem("📝 About"): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") # About tab with gr.TabItem("║", interactive=False): gr.Markdown("", elem_classes="markdown-text") # Task-specific leaderboards for task, metadata in TASK_METADATA_MULTIPLECHOICE.items(): with gr.TabItem(f"{metadata['icon']}{task}"): task_description = TASK_DESCRIPTIONS.get(task, "Description not available.") gr.Markdown(task_description, elem_classes="markdown-text") leaderboard = update_task_leaderboard( LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Prompt Std": "Prompt Std", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}), default_selection=['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'], hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']] ) # About tab with gr.TabItem("│", interactive=False): gr.Markdown("", elem_classes="markdown-text") # Task-specific leaderboards for task, metadata in TASK_METADATA_GENERATIVE.items(): with gr.TabItem(f"{metadata['icon']}{task}"): task_description = TASK_DESCRIPTIONS.get(task, "Description not available.") gr.Markdown(task_description, elem_classes="markdown-text") leaderboard = update_task_leaderboard( LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Prompt Std": "Prompt Std", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}), default_selection=['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'], hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']] ) # Citation section with gr.Accordion("📙 Citation", open=False): gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True) with gr.Accordion("📙 Credits", open=False): gr.Markdown( """ **This project has benefited from the following support:** - 🧠 **Codebase**: Based on and extended from the Open Italian LLM Leaderboard, developed by **Alessandro Ercolani** and **Samuele Colombo**. We warmly thank them for their invaluable support and guidance in implementing this leaderboard. - 💶 **Funding**: Partially supported by the PNRR project **FAIR - Future AI Research (PE00000013)**, under the NRRP MUR program funded by **NextGenerationEU**. - 🖥️ **Computation**: We gratefully acknowledge **CINECA** for granting access to the **LEONARDO** supercomputer. """ ) # Background job to restart space scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() # Launch the app with concurrent queueing demo.queue(default_concurrency_limit=40).launch(debug=True, # Enable Gradio debug mode show_error=True)