Upload 4 files

app.py

@@ -0,0 +1,118 @@
+import gradio as gr
+import pandas as pd
+import json
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS, LEADERBOARD_CSS
+from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
+from datetime import datetime, timezone
+
+LAST_UPDATED = "Nov 22th 2024"
+
+column_names = {
+    "model": "Model",
+    "Average WER ⬇️": "Average WER ⬇️",    
+    "RTFx": "RTFx ⬆️️",
+    "Bulgarian_female": "Bulgarian female",
+    "Bulgarian_male": "Bulgarian male",
+    "Catalan_female": "Catalan female",
+    "Chinese_female": "Chinese female",
+    "Chinese_male": "Chinese male",
+    "Eastern_European_male": "Eastern European male",
+    "European_male": "European male",
+    "French_female": "French female",
+    "Ghanain_English_female": "Ghanain English female",
+    "Indian_English_female": "Indian English female",
+    "Indian_English_male": "Indian English male",
+    "Indonesian_female": "Indonesian female",
+    "Irish_English_female": "Irish English female",
+    "Irish_English_male": "Irish English male",
+    "Israeli_male": "Israeli male",
+    "Italian_female": "Italian female",
+    "Jamaican_English_female": "Jamaican English female",
+    "Jamaican_English_male": "Jamaican English male",
+    "Kenyan_English_female": "Kenyan English female",
+    "Kenyan_English_male": "Kenyan English male",
+    "Latin_American_female": "Latin American female",
+    "Latin_American_male": "Latin American male",
+    "Lithuanian_male": "Lithuanian male",
+    "Mainstream_US_English_female": "Mainstream US English female",
+    "Mainstream_US_English_male": "Mainstream US English male",
+    "Nigerian_English_female": "Nigerian English female",
+    "Nigerian_English_male": "Nigerian English male",
+    "Romanian_female": "Romanian female",
+    "Scottish_English_male": "Scottish English male",
+    "Southern_British_English_male": "Southern British English male",
+    "Spanish_female": "Spanish female",
+    "Spanish_male": "Spanish male",
+    "Vietnamese_female": "Vietnamese female",
+    "Vietnamese_male": "Vietnamese male",
+}
+
+eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+
+if not csv_results.exists():
+    raise Exception(f"CSV file {csv_results} does not exist locally")
+    
+# Get csv with data and parse columns
+original_df = pd.read_csv(csv_results)
+
+# Formats the columns
+def formatter(x):
+    if type(x) is str:
+        x = x
+    else: 
+        x = round(x, 2)
+    return x
+
+for col in original_df.columns:
+    if col == "model":
+        original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+    else:
+        original_df[col] = original_df[col].apply(formatter) # For numerical values
+ 
+original_df.rename(columns=column_names, inplace=True)
+original_df.sort_values(by='Average WER ⬇️', inplace=True)
+
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+
+with gr.Blocks(css=LEADERBOARD_CSS) as demo:
+    # gr.HTML(BANNER, elem_id="banner")
+    # Write a header with the title
+    gr.Markdown("<h1>🤗 Open Automatic Speech Recognition on EdAcc Dataset</h1>", elem_classes="markdown-text")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
+            # Add column filter dropdown
+            column_filter = gr.Dropdown(
+                choices=["All"] + [v for k,v in column_names.items() if k != "model"],
+                label="Filter by column", 
+                multiselect=True,
+                value=["All"],
+                elem_id="column-filter"
+            )
+            
+            leaderboard_table = gr.components.Dataframe(
+                value=original_df,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+            
+            # Update table columns when filter changes
+            def update_table(cols):
+                if "All" in cols:
+                    return gr.Dataframe(value=original_df)
+                selected_cols = ["Model"] + cols  # Always include the Model column
+                return gr.Dataframe(value=original_df[selected_cols])
+            
+            column_filter.change(
+                fn=update_table,
+                inputs=[column_filter],
+                outputs=[leaderboard_table]
+            )
+
+demo.launch(ssr_mode=False)

constants.py

@@ -0,0 +1,119 @@
+from pathlib import Path
+
+# Directory where request by models are stored
+DIR_OUTPUT_REQUESTS = Path("requested_models")
+EVAL_REQUESTS_PATH = Path("eval_requests")
+
+##########################
+# Text definitions       #
+##########################
+
+banner_url = "https://huggingface.co/datasets/reach-vb/random-images/resolve/main/asr_leaderboard.png"
+BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
+
+TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Open Automatic Speech Recognition Leaderboard </b> </body> </html>"
+
+INTRODUCTION_TEXT = "📐 Results on [EdAcc Dataset](https://huggingface.co/datasets/edinburghcstr/edacc) split by accents and gender. \
+    \nWe report the Average [WER](https://huggingface.co/spaces/evaluate-metric/wer) (⬇️ lower the better) and [RTFx](https://github.com/NVIDIA/DeepLearningExamples/blob/master/Kaldi/SpeechRecognition/README.md#metrics) (⬆️ higher the better)."
+
+
+CITATION_TEXT = """@misc{open-asr-leaderboard,
+	title        = {Open Automatic Speech Recognition Leaderboard},
+	author       = {Srivastav, Vaibhav and Majumdar, Somshubra and Koluguri, Nithin and Moumen, Adel and Gandhi, Sanchit and others},
+	year         = 2023,
+	publisher    = {Hugging Face},
+	howpublished = "\\url{https://huggingface.co/spaces/hf-audio/open_asr_leaderboard}"
+}
+"""
+
+METRICS_TAB_TEXT = """
+Here you will find details about the speech recognition metrics and datasets reported in our leaderboard.
+
+## Metrics
+
+Models are evaluated jointly using the Word Error Rate (WER) and Inverse Real Time Factor (RTFx) metrics. The WER metric
+is used to assess the accuracy of a system, and the RTFx the inference speed. Models are ranked in the leaderboard based 
+on their WER, lowest to highest.
+
+Crucially, the WER and RTFx values are computed for the same inference run using a single script. The implication of this is two-fold:
+1. The WER and RTFx values are coupled: for a given WER, one can expect to achieve the corresponding RTFx. This allows the proposer to trade-off lower WER for higher RTFx should they wish.
+2. The WER and RTFx values are averaged over all audios in the benchmark (in the order of thousands of audios).
+
+For details on reproducing the benchmark numbers, refer to the [Open ASR GitHub repository](https://github.com/huggingface/open_asr_leaderboard#evaluate-a-model).
+
+### Word Error Rate (WER)
+
+Word Error Rate is used to measure the **accuracy** of automatic speech recognition systems. It calculates the percentage 
+of words in the system's output that differ from the reference (correct) transcript. **A lower WER value indicates higher accuracy**.
+
+Take the following example:
+
+| Reference:  | the | cat | sat     | on  | the | mat |
+|-------------|-----|-----|---------|-----|-----|-----|
+| Prediction: | the | cat | **sit** | on  | the |     |  |
+| Label:      | ✅   | ✅   | S       | ✅   | ✅   | D   |
+
+Here, we have:
+* 1 substitution ("sit" instead of "sat")
+* 0 insertions
+* 1 deletion ("mat" is missing)
+
+This gives 2 errors in total. To get our word error rate, we divide the total number of errors (substitutions + insertions + deletions) by the total number of words in our
+reference (N), which for this example is 6:
+
+```
+WER = (S + I + D) / N = (1 + 0 + 1) / 6 = 0.333
+```
+
+Giving a WER of 0.33, or 33%. For a fair comparison, we calculate **zero-shot** (i.e. pre-trained models only) *normalised WER* for all the model checkpoints, meaning punctuation and casing is removed from the references and predictions. You can find the evaluation code on our [Github repository](https://github.com/huggingface/open_asr_leaderboard). To read more about how the WER is computed, refer to the [Audio Transformers Course](https://huggingface.co/learn/audio-course/chapter5/evaluation).
+
+### Inverse Real Time Factor (RTFx)
+
+Inverse Real Time Factor is a measure of  the **latency** of automatic speech recognition systems, i.e. how long it takes an 
+model to process a given amount of speech. It is defined as:
+```
+RTFx = (number of seconds of audio inferred) / (compute time in seconds)
+``` 
+
+Therefore, and RTFx of 1 means a system processes speech as fast as it's spoken, while an RTFx of 2 means it takes half the time. 
+Thus, **a higher RTFx value indicates lower latency**.
+
+## How to reproduce our results
+
+The ASR Leaderboard will be a continued effort to benchmark open source/access speech recognition models where possible. 
+Along with the Leaderboard we're open-sourcing the codebase used for running these evaluations.
+For more details head over to our repo at: https://github.com/huggingface/open_asr_leaderboard 
+
+P.S. We'd love to know which other models you'd like us to benchmark next. Contributions are more than welcome! ♥️
+
+## Benchmark datasets
+
+Evaluating Speech Recognition systems is a hard problem. We use the multi-dataset benchmarking strategy proposed in the 
+[ESB paper](https://arxiv.org/abs/2210.13352) to obtain robust evaluation scores for each model.
+
+ESB is a benchmark for evaluating the performance of a single automatic speech recognition (ASR) system across a broad 
+set of speech datasets. It comprises eight English speech recognition datasets, capturing a broad range of domains, 
+acoustic conditions, speaker styles, and transcription requirements. As such, it gives a better indication of how 
+a model is likely to perform on downstream ASR compared to evaluating it on one dataset alone.
+
+The ESB score is calculated as a macro-average of the WER scores across the ESB datasets. The models in the leaderboard
+are ranked based on their average WER scores, from lowest to highest.
+
+| Dataset                                                                                 | Domain                      | Speaking Style        | Train (h) | Dev (h) | Test (h) | Transcriptions     | License         |
+|-----------------------------------------------------------------------------------------|-----------------------------|-----------------------|-----------|---------|----------|--------------------|-----------------|
+| [LibriSpeech](https://huggingface.co/datasets/librispeech_asr)                          | Audiobook                   | Narrated              | 960       | 11      | 11       | Normalised         | CC-BY-4.0       |
+| [VoxPopuli](https://huggingface.co/datasets/facebook/voxpopuli)                         | European Parliament         | Oratory               | 523       | 5       | 5        | Punctuated         | CC0             |
+| [TED-LIUM](https://huggingface.co/datasets/LIUM/tedlium)                                | TED talks                   | Oratory               | 454       | 2       | 3        | Normalised         | CC-BY-NC-ND 3.0 |
+| [GigaSpeech](https://huggingface.co/datasets/speechcolab/gigaspeech)                    | Audiobook, podcast, YouTube | Narrated, spontaneous | 2500      | 12      | 40       | Punctuated         | apache-2.0      |
+| [SPGISpeech](https://huggingface.co/datasets/kensho/spgispeech)                         | Financial meetings          | Oratory, spontaneous  | 4900      | 100     | 100      | Punctuated & Cased | User Agreement  |
+| [Earnings-22](https://huggingface.co/datasets/revdotcom/earnings22)                     | Financial meetings          | Oratory, spontaneous  | 105       | 5       | 5        | Punctuated & Cased | CC-BY-SA-4.0    |
+| [AMI](https://huggingface.co/datasets/edinburghcstr/ami)                                | Meetings                    | Spontaneous           | 78        | 9       | 9        | Punctuated & Cased | CC-BY-4.0       |
+
+For more details on the individual datasets and how models are evaluated to give the ESB score, refer to the [ESB paper](https://arxiv.org/abs/2210.13352).
+"""
+
+LEADERBOARD_CSS = """
+#leaderboard-table th .header-content {
+    white-space: nowrap;
+}
+"""

init.py

@@ -0,0 +1,92 @@
+import os 
+from constants import EVAL_REQUESTS_PATH
+from pathlib import Path
+from huggingface_hub import HfApi, Repository
+
+TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
+QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
+QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
+
+hf_api = HfApi(
+    endpoint="https://huggingface.co", 
+    token=TOKEN_HUB, 
+)
+
+def load_all_info_from_dataset_hub():
+    eval_queue_repo = None
+    requested_models = None
+
+    passed = True
+    if TOKEN_HUB is None:
+        passed = False
+    else:
+        print("Pulling evaluation requests and results.")
+
+        eval_queue_repo = Repository(
+            local_dir=QUEUE_PATH,
+            clone_from=QUEUE_REPO,
+            use_auth_token=TOKEN_HUB,
+            repo_type="dataset",
+        )
+        eval_queue_repo.git_pull()
+        
+        # Local directory where dataset repo is cloned + folder with eval requests
+        directory = QUEUE_PATH / EVAL_REQUESTS_PATH
+        requested_models = get_all_requested_models(directory)
+        requested_models = [p.stem for p in requested_models]
+        # Local directory where dataset repo is cloned
+        csv_results = get_csv_with_results(QUEUE_PATH)
+        if csv_results is None:
+            passed = False
+    if not passed:
+        raise ValueError("No Hugging Face token provided. Skipping evaluation requests and results.")
+
+    return eval_queue_repo, requested_models, csv_results
+
+
+def upload_file(requested_model_name, path_or_fileobj):
+    dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
+    dest_repo_file = str(dest_repo_file)
+    hf_api.upload_file(
+            path_or_fileobj=path_or_fileobj,
+            path_in_repo=str(dest_repo_file),
+            repo_id=QUEUE_REPO,
+            token=TOKEN_HUB,
+            repo_type="dataset",
+            commit_message=f"Add {requested_model_name} to eval queue")
+
+def get_all_requested_models(directory):
+    directory = Path(directory)
+    all_requested_models = list(directory.glob("*.txt"))
+    return all_requested_models
+
+def get_csv_with_results(directory):
+    directory = Path(directory)
+    all_csv_files = list(directory.glob("*.csv"))
+    latest = [f for f in all_csv_files if f.stem.endswith("latest")]
+    if len(latest) != 1:
+        return None
+    return latest[0]
+
+
+
+def is_model_on_hub(model_name, revision="main") -> bool:
+    try:
+        model_name = model_name.replace(" ","")
+        author = model_name.split("/")[0]
+        model_id = model_name.split("/")[1]
+        if len(author) == 0 or len(model_id) == 0:
+            return False, "is not a valid model name. Please use the format `author/model_name`."
+    except Exception as e:
+        return False, "is not a valid model name. Please use the format `author/model_name`."
+
+    try:
+        models = list(hf_api.list_models(author=author, search=model_id))
+        matched = [model_name for m in models if m.modelId == model_name]
+        if len(matched) != 1:
+            return False, "was not found on the hub!"
+        else:
+            return True, None
+    except Exception as e:
+        print(f"Could not get the model from the hub.: {e}")
+        return False, "was not found on hub!"

utils_display.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown")
+    avg_wer = ColumnContent("Average WER ⬇️", "number")    
+    rtf = ColumnContent("RTFx ⬆️️", "number")
+    bulgarian_female = ColumnContent("Bulgarian female", "number")
+    bulgarian_male = ColumnContent("Bulgarian male", "number")
+    catalan_female = ColumnContent("Catalan female", "number")
+    chinese_female = ColumnContent("Chinese female", "number")
+    chinese_male = ColumnContent("Chinese male", "number")
+    eastern_european_male = ColumnContent("Eastern European male", "number")
+    european_male = ColumnContent("European male", "number")
+    french_female = ColumnContent("French female", "number")
+    ghanain_english_female = ColumnContent("Ghanain English female", "number")
+    indian_english_female = ColumnContent("Indian English female", "number")
+    indian_english_male = ColumnContent("Indian English male", "number")
+    indonesian_female = ColumnContent("Indonesian female", "number")
+    irish_english_female = ColumnContent("Irish English female", "number")
+    irish_english_male = ColumnContent("Irish English male", "number")
+    israeli_male = ColumnContent("Israeli male", "number")
+    italian_female = ColumnContent("Italian female", "number")
+    jamaican_english_female = ColumnContent("Jamaican English female", "number")
+    jamaican_english_male = ColumnContent("Jamaican English male", "number")
+    kenyan_english_female = ColumnContent("Kenyan English female", "number")
+    kenyan_english_male = ColumnContent("Kenyan English male", "number")
+    latin_american_female = ColumnContent("Latin American female", "number")
+    latin_american_male = ColumnContent("Latin American male", "number")
+    lithuanian_male = ColumnContent("Lithuanian male", "number")
+    mainstream_us_english_female = ColumnContent("Mainstream US English female", "number")
+    mainstream_us_english_male = ColumnContent("Mainstream US English male", "number")
+    nigerian_english_female = ColumnContent("Nigerian English female", "number")
+    nigerian_english_male = ColumnContent("Nigerian English male", "number")
+    romanian_female = ColumnContent("Romanian female", "number")
+    scottish_english_male = ColumnContent("Scottish English male", "number")
+    southern_british_english_male = ColumnContent("Southern British English male", "number")
+    spanish_female = ColumnContent("Spanish female", "number")
+    spanish_male = ColumnContent("Spanish male", "number")
+    vietnamese_female = ColumnContent("Vietnamese female", "number")
+    vietnamese_male = ColumnContent("Vietnamese male", "number")
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"