import gradio as gr import pandas as pd import json import os from constants import LEADERBOARD_CSS, EXPLANATION, EXPLANATION_EDACC, EXPLANATION_AFRI from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message from datetime import datetime, timezone from huggingface_hub import HfApi, upload_file LAST_UPDATED = "Nov 22th 2024" column_names = { "model": "Model", "Average WER ⬇️": "Average WER ⬇️", "Average Female WER ⬇️": "Average Female WER ⬇️", "Average Male WER ⬇️": "Average Male WER ⬇️", "RTFx": "RTFx ⬆️️", "Bulgarian_female": "Bulgarian female", "Bulgarian_male": "Bulgarian male", "Catalan_female": "Catalan female", "Chinese_female": "Chinese female", "Chinese_male": "Chinese male", "Eastern_European_male": "Eastern European male", "European_male": "European male", "French_female": "French female", "Ghanain_English_female": "Ghanain English female", "Indian_English_female": "Indian English female", "Indian_English_male": "Indian English male", "Indonesian_female": "Indonesian female", "Irish_English_female": "Irish English female", "Irish_English_male": "Irish English male", "Israeli_male": "Israeli male", "Italian_female": "Italian female", "Jamaican_English_female": "Jamaican English female", "Jamaican_English_male": "Jamaican English male", "Kenyan_English_female": "Kenyan English female", "Kenyan_English_male": "Kenyan English male", "Latin_American_female": "Latin American female", "Latin_American_male": "Latin American male", "Lithuanian_male": "Lithuanian male", "Mainstream_US_English_female": "Mainstream US English female", "Mainstream_US_English_male": "Mainstream US English male", "Nigerian_English_female": "Nigerian English female", "Nigerian_English_male": "Nigerian English male", "Romanian_female": "Romanian female", "Scottish_English_male": "Scottish English male", "Southern_British_English_male": "Southern British English male", "Spanish_female": "Spanish female", "Spanish_male": "Spanish male", "Vietnamese_female": "Vietnamese female", "Vietnamese_male": "Vietnamese male", "agatu_test": "Agatu", "angas_test": "Angas", "bajju_test": "Bajju", "bini_test": "Bini", "brass_test": "Brass", "delta_test": "Delta", "eggon_test": "Eggon", "ekene_test": "Ekene", "ekpeye_test": "Ekpeye", "gbagyi_test": "Gbagyi", "igarra_test": "Igarra", "ijaw-nembe_test": "Ijaw-Nembe", "ikulu_test": "Ikulu", "jaba_test": "Jaba", "jukun_test": "Jukun", "khana_test": "Khana", "mada_test": "Mada", "mwaghavul_test": "Mwaghavul", "ukwuani_test": "Ukwuani", "yoruba-hausa_test": "Yoruba-Hausa", } african_cols = ["Ghanain English female", "Kenyan English female", "Kenyan English male", "Nigerian English female", "Nigerian English male"] north_american_cols = ["Mainstream US English female", "Mainstream US English male"] caribbean_cols = ["Jamaican English female", "Jamaican English male"] latin_american_cols = ["Latin American female", "Latin American male"] british_cols = ["Irish English female", "Irish English male", "Scottish English male", "Southern British English male"] european_cols = ["Eastern European male", "European male", "French female", "Italian female", "Spanish female", "Spanish male", "Catalan female", "Bulgarian female", "Bulgarian male", "Lithuanian male", "Romanian female"] asian_cols = ["Chinese female", "Chinese male", "Indonesian female", "Vietnamese female", "Vietnamese male", "Indian English female", "Indian English male"] eval_queue_repo_edacc, requested_models, csv_results_edacc, csv_results_afrispeech = load_all_info_from_dataset_hub() if not csv_results_edacc.exists(): raise Exception(f"CSV file {csv_results_edacc} does not exist locally") # Get csv with data and parse columns original_df = pd.read_csv(csv_results_edacc) afrispeech_df = pd.read_csv(csv_results_afrispeech) # Formats the columns def formatter(x): if type(x) is str: x = x else: x = round(x, 2) return x for col in original_df.columns: if col == "model": original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x))) else: original_df[col] = original_df[col].apply(formatter) # For numerical values for col in afrispeech_df.columns: if col == "model": afrispeech_df[col] = afrispeech_df[col].apply(lambda x: x.replace(x, make_clickable_model(x))) else: afrispeech_df[col] = afrispeech_df[col].apply(formatter) # For numerical values original_df.rename(columns=column_names, inplace=True) original_df.sort_values(by='Average WER ⬇️', inplace=True) afrispeech_df.rename(columns=column_names, inplace=True) afrispeech_df.sort_values(by='Average WER ⬇️', inplace=True) female_cols = [col for col in original_df.columns if 'female' == col.split(' ')[-1]] male_cols = [col for col in original_df.columns if 'male' == col.split(' ')[-1]] # Create male DataFrame properly male_df = original_df[['Model'] + male_cols].copy() # Create explicit copy with model column male_df.loc[:, 'Average Male WER ⬇️'] = male_df[male_cols].mean(axis=1) male_df.loc[:, 'Average Male WER ⬇️'] = male_df['Average Male WER ⬇️'].apply(formatter) male_df = male_df[['Model', 'Average Male WER ⬇️'] + male_cols] # Create female DataFrame properly female_df = original_df[['Model'] + female_cols].copy() # Create explicit copy with model column female_df.loc[:, 'Average Female WER ⬇️'] = female_df[female_cols].mean(axis=1) female_df.loc[:, 'Average Female WER ⬇️'] = female_df['Average Female WER ⬇️'].apply(formatter) female_df = female_df[['Model', 'Average Female WER ⬇️'] + female_cols] african_df = original_df[['Model'] + african_cols].copy() african_df.loc[:, 'Average African WER ⬇️'] = african_df[african_cols].mean(axis=1) african_df.loc[:, 'Average African WER ⬇️'] = african_df['Average African WER ⬇️'].apply(formatter) african_df = african_df[['Model', 'Average African WER ⬇️'] + african_cols] north_american_df = original_df[['Model'] + north_american_cols].copy() north_american_df.loc[:, 'Average North American WER ⬇️'] = north_american_df[north_american_cols].mean(axis=1) north_american_df.loc[:, 'Average North American WER ⬇️'] = north_american_df['Average North American WER ⬇️'].apply(formatter) north_american_df = north_american_df[['Model', 'Average North American WER ⬇️'] + north_american_cols] caribbean_df = original_df[['Model'] + caribbean_cols].copy() caribbean_df.loc[:, 'Average Caribbean WER ⬇️'] = caribbean_df[caribbean_cols].mean(axis=1) caribbean_df.loc[:, 'Average Caribbean WER ⬇️'] = caribbean_df['Average Caribbean WER ⬇️'].apply(formatter) caribbean_df = caribbean_df[['Model', 'Average Caribbean WER ⬇️'] + caribbean_cols] latin_american_df = original_df[['Model'] + latin_american_cols].copy() latin_american_df.loc[:, 'Average Latin American WER ⬇️'] = latin_american_df[latin_american_cols].mean(axis=1) latin_american_df.loc[:, 'Average Latin American WER ⬇️'] = latin_american_df['Average Latin American WER ⬇️'].apply(formatter) latin_american_df = latin_american_df[['Model', 'Average Latin American WER ⬇️'] + latin_american_cols] british_df = original_df[['Model'] + british_cols].copy() british_df.loc[:, 'Average British WER ⬇️'] = british_df[british_cols].mean(axis=1) british_df.loc[:, 'Average British WER ⬇️'] = british_df['Average British WER ⬇️'].apply(formatter) british_df = british_df[['Model', 'Average British WER ⬇️'] + british_cols] european_df = original_df[['Model'] + european_cols].copy() european_df.loc[:, 'Average European WER ⬇️'] = european_df[european_cols].mean(axis=1) european_df.loc[:, 'Average European WER ⬇️'] = european_df['Average European WER ⬇️'].apply(formatter) european_df = european_df[['Model', 'Average European WER ⬇️'] + european_cols] asian_df = original_df[['Model'] + asian_cols].copy() asian_df.loc[:, 'Average Asian WER ⬇️'] = asian_df[asian_cols].mean(axis=1) asian_df.loc[:, 'Average Asian WER ⬇️'] = asian_df['Average Asian WER ⬇️'].apply(formatter) asian_df = asian_df[['Model', 'Average Asian WER ⬇️'] + asian_cols] # add average female and mal to original df and place it after average wer original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average African WER ⬇️', african_df['Average African WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average North American WER ⬇️', north_american_df['Average North American WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Caribbean WER ⬇️', caribbean_df['Average Caribbean WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Latin American WER ⬇️', latin_american_df['Average Latin American WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average British WER ⬇️', british_df['Average British WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average European WER ⬇️', european_df['Average European WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Asian WER ⬇️', asian_df['Average Asian WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Female WER ⬇️', female_df['Average Female WER ⬇️']) original_df.insert(original_df.columns.get_loc('Average WER ⬇️') + 1, 'Average Male WER ⬇️', male_df['Average Male WER ⬇️']) # Save the updated DataFrame to a temporary CSV file timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") # Generate a timestamp temp_csv_filename = f"updated_leaderboard_{timestamp}.csv" # Create a unique filename original_df.to_csv(temp_csv_filename, index=False) # Save the DataFrame to a temporary CSV file # Upload the CSV file to Hugging Face hf_api = HfApi() repo_id = "Steveeeeeeen/whisper-leaderboard-evals" # Replace with your Hugging Face repo ID TOKEN_HUB = os.environ.get("TOKEN_HUB", None) upload_file( path_or_fileobj=temp_csv_filename, path_in_repo=f"data/{temp_csv_filename}", # Path in the Hugging Face repo repo_id=repo_id, token=TOKEN_HUB, repo_type="dataset" ) print(f"Updated leaderboard uploaded to Hugging Face: {repo_id}/data/{temp_csv_filename}") COLS = [c.name for c in fields(AutoEvalColumn)] TYPES = [c.type for c in fields(AutoEvalColumn)] with gr.Blocks(css=LEADERBOARD_CSS) as demo: # gr.HTML(BANNER, elem_id="banner") # Write a header with the title gr.Markdown("

🤫 How Biased is Whisper?

", elem_classes="markdown-text") gr.Markdown(EXPLANATION, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Edacc Results", elem_id="od-benchmark-tab-table", id=0): gr.Markdown(EXPLANATION_EDACC, elem_classes="markdown-text") # Add column filter dropdown column_filter = gr.Dropdown( choices=["All", "Female", "Male", "African", "North American", "Caribbean", "Latin American", "British", "European", "Asian"] + [v for k,v in column_names.items() if k != "model"], label="Filter by column", multiselect=True, value=["All"], elem_id="column-filter" ) leaderboard_table = gr.components.Dataframe( value=original_df, datatype=TYPES, elem_id="leaderboard-table", interactive=False, visible=True, ) # Update table columns when filter changes def update_table(cols): # Dictionary mapping view names to their corresponding dataframes view_mapping = { "All": original_df, "Female": female_df, "Male": male_df, "African": african_df, "North American": north_american_df, "Caribbean": caribbean_df, "Latin American": latin_american_df, "British": british_df, "European": european_df, "Asian": asian_df } # Handle special views selected_special_views = [view for view in view_mapping.keys() if view in cols] if selected_special_views: # Start with the first selected view's columns result_cols = set(view_mapping[selected_special_views[0]].columns) # Take union of columns from all selected views for view in selected_special_views[1:]: result_cols.update(view_mapping[view].columns) # Convert to list and ensure "Model" is first result_cols = ["Model"] + sorted(list(result_cols - {"Model"})) # Merge all relevant columns from original_df return gr.Dataframe(value=original_df[result_cols]) # If no special view is selected, return filtered columns from original df selected_cols = ["Model"] + cols # Always include the Model column return gr.Dataframe(value=original_df[selected_cols]) column_filter.change( fn=update_table, inputs=[column_filter], outputs=[leaderboard_table] ) with gr.TabItem("🏅 Afrispeech Results", elem_id="od-benchmark-tab-table", id=1): gr.Markdown(EXPLANATION_AFRI, elem_classes="markdown-text") # Add column filter dropdown afrispeech_column_filter = gr.Dropdown( choices=["All"] + [v for k,v in column_names.items() if k != "model" and v in afrispeech_df.columns], label="Filter by column", multiselect=True, value=["All"], elem_id="afrispeech-column-filter" ) leaderboard_table = gr.components.Dataframe( value=afrispeech_df, datatype=TYPES, elem_id="leaderboard-table", interactive=False, visible=True, ) # Update table columns when filter changes def update_afrispeech_table(cols): if "All" in cols: return gr.Dataframe(value=afrispeech_df) selected_cols = ["Model"] + cols # Always include the Model column return gr.Dataframe(value=afrispeech_df[selected_cols]) afrispeech_column_filter.change( fn=update_afrispeech_table, inputs=[afrispeech_column_filter], outputs=[leaderboard_table] ) demo.launch(ssr_mode=False)