Initial commit

README.md

@@ -1,12 +1,12 @@
 ---
-title: Open Asr Leaderboard
-emoji: 😻
+title: Open ASR Leaderboard
+emoji: 🏆
 colorFrom: red
 colorTo: blue
 sdk: gradio
 sdk_version: 3.42.0
 app_file: app.py
-pinned: false
+pinned: true
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,145 @@
+import gradio as gr
+import pandas as pd
+import json
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS
+from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
+from datetime import datetime, timezone
+
+LAST_UPDATED = "Sep 7th 2023"
+
+column_names = {
+    "MODEL": "Model",
+    "RTF": "RTF (1e-3) ⬇️",
+    "Avg. WER": "Average WER ⬇️",
+    "AMI WER": "AMI",
+    "Earnings22 WER": "Earnings22",
+    "Gigaspeech WER": "Gigaspeech",
+    "LS Clean WER": "LS Clean",
+    "LS Other WER": "LS Other",
+    "SPGISpeech WER": "SPGISpeech",
+    "Tedlium WER": "Tedlium",
+    "Voxpopuli WER": "Voxpopuli",
+    "Common Voice WER": "Common Voice"}
+
+eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+
+if not csv_results.exists():
+    raise Exception(f"CSV file {csv_results} does not exist locally")
+    
+# Get csv with data and parse columns
+original_df = pd.read_csv(csv_results)
+
+# Formats the columns
+def formatter(x):
+    x = round(x, 2)
+    return x
+
+for col in original_df.columns:
+    if col == "model":
+        original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+    else:
+        original_df[col] = original_df[col].apply(formatter) # For numerical values
+ 
+original_df.rename(columns=column_names, inplace=True)
+original_df.sort_values(by='Average WER ⬇️', inplace=True)
+
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+
+def request_model(model_text, chbcoco2017):
+    
+    # Determine the selected checkboxes
+    dataset_selection = []
+    if chbcoco2017:
+        dataset_selection.append("ESB Datasets tests only")
+
+    if len(dataset_selection) == 0:
+        return styled_error("You need to select at least one dataset")
+        
+    base_model_on_hub, error_msg = is_model_on_hub(model_text)
+
+    if not base_model_on_hub:
+        return styled_error(f"Base model '{model_text}' {error_msg}")
+    
+    # Construct the output dictionary
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    required_datasets = ', '.join(dataset_selection)
+    eval_entry = {
+        "date": current_time,
+        "model": model_text,
+        "datasets_selected": required_datasets
+    }
+    
+    # Prepare file path 
+    DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True)
+    
+    fn_datasets = '@ '.join(dataset_selection)
+    filename = model_text.replace("/","@") + "@@" + fn_datasets 
+    if filename in requested_models:
+        return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.")
+    try:
+        filename_ext = filename + ".txt"
+        out_filepath = DIR_OUTPUT_REQUESTS / filename_ext
+
+        # Write the results to a text file
+        with open(out_filepath, "w") as f:
+            f.write(json.dumps(eval_entry))
+            
+        upload_file(filename, out_filepath)
+        
+        # Include file in the list of uploaded files
+        requested_models.append(filename)
+        
+        # Remove the local file
+        out_filepath.unlink()
+
+        return styled_message("🤗 Your request has been submitted and will be evaluated soon!</p>")
+    except Exception as e:
+        return styled_error(f"Error submitting request!")
+
+with gr.Blocks() as demo:
+    gr.HTML(BANNER, elem_id="banner")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
+            leaderboard_table = gr.components.Dataframe(
+                value=original_df,
+                datatype=TYPES,
+                max_rows=None,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                )
+
+        with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
+            gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=2):
+            with gr.Column():
+                gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
+            with gr.Column():
+                gr.Markdown("Select a dataset:", elem_classes="markdown-text")
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)")
+                    chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset", visible=False, value=True, interactive=False)
+                with gr.Column():
+                    mdw_submission_result = gr.Markdown()
+                    btn_submitt = gr.Button(value="🚀 Request")
+                    btn_submitt.click(request_model, 
+                                      [model_name_textbox, chb_coco2017], 
+                                      mdw_submission_result)
+
+    gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")
+    
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            gr.Textbox(
+                value=CITATION_TEXT, lines=7,
+                label="Copy the BibTeX snippet to cite this source",
+                elem_id="citation-button",
+            ).style(show_copy_button=True)
+
+demo.launch()

constants.py

@@ -0,0 +1,98 @@
+from pathlib import Path
+
+# Directory where request by models are stored
+DIR_OUTPUT_REQUESTS = Path("requested_models")
+EVAL_REQUESTS_PATH = Path("eval_requests")
+
+##########################
+# Text definitions       #
+##########################
+
+banner_url = "https://huggingface.co/datasets/reach-vb/random-images/resolve/main/asr_leaderboard.png"
+BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
+
+TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Open Automatic Speech Recognition Leaderboard </b> </body> </html>"
+
+INTRODUCTION_TEXT = "📐 The 🤗 Open ASR Leaderboard ranks and evaluates speech recognition models \
+    on the Hugging Face Hub. \
+    \nWe report the Average [WER](https://huggingface.co/spaces/evaluate-metric/wer) (⬇️) and [RTF](https://openvoice-tech.net/index.php/Real-time-factor) (⬇️) - lower the better. Models are ranked based on their Average WER, from lowest to highest. Check the 📈 Metrics tab to understand how the models are evaluated. \
+    \nIf you want results for a model that is not listed here, you can submit a request for it to be included ✉️✨. \
+    \nThe leaderboard currently focuses on English speech recognition, and will be expanded to multilingual evaluation in later versions."
+
+CITATION_TEXT = """@misc{open-asr-leaderboard,
+	title        = {Open Automatic Speech Recognition Leaderboard},
+	author       = {Srivastav, Vaibhav and Majumdar, Somshubra and Koluguri, Nithin and Moumen, Adel and Gandhi, Sanchit and Hugging Face Team and Nvidia NeMo Team and SpeechBrain Team},
+	year         = 2023,
+	publisher    = {Hugging Face},
+	howpublished = "\\url{https://huggingface.co/spaces/huggingface.co/spaces/open-asr-leaderboard/leaderboard}"
+}
+"""
+
+METRICS_TAB_TEXT = """
+Here you will find details about the speech recognition metrics and datasets reported in our leaderboard.
+
+## Metrics
+
+🎯 Word Error Rate (WER) and Real-Time Factor (RTF) are popular metrics for evaluating the accuracy of speech recognition 
+models by estimating how accurate the predictions from the models are and how fast they are returned. We explain them each
+below.
+
+### Word Error Rate (WER)
+
+Word Error Rate is used to measure the **accuracy** of automatic speech recognition systems. It calculates the percentage 
+of words in the system's output that differ from the reference (correct) transcript. **A lower WER value indicates higher accuracy**.
+
+```
+Example: If the reference transcript is "I really love cats," and the ASR system outputs "I don't love dogs,".
+The WER would be `50%` because 2 out of 4 words are incorrect.
+```
+
+For a fair comparison, we calculate **zero-shot** (i.e. pre-trained models only) *normalised WER* for all the model checkpoints. You can find the evaluation code on our [Github repository](https://github.com/huggingface/open_asr_leaderboard). To read more about how the WER is computed, refer to the [Audio Transformers Course](https://huggingface.co/learn/audio-course/chapter5/evaluation).
+
+### Real Time Factor (RTF)
+
+Real Time Factor is a measure of  the **latency** of automatic speech recognition systems, i.e. how long it takes an 
+model to process a given amount of speech. It's usually expressed as a multiple of real time. An RTF of 1 means it processes 
+speech as fast as it's spoken, while an RTF of 2 means it takes twice as long. Thus, **a lower RTF value indicates lower latency**.
+
+```
+Example: If it takes an ASR system 10 seconds to transcribe 10 seconds of speech, the RTF is 1. 
+If it takes 20 seconds to transcribe the same 10 seconds of speech, the RTF is 2.
+```
+
+For the benchmark, we report RTF averaged over a 10 minute audio sample with 5 warm up batches followed 3 graded batches.
+
+## How to reproduce our results
+
+The ASR Leaderboard will be a continued effort to benchmark open source/access speech recognition models where possible. 
+Along with the Leaderboard we're open-sourcing the codebase used for running these evaluations.
+For more details head over to our repo at: https://github.com/huggingface/open_asr_leaderboard 
+
+P.S. We'd love to know which other models you'd like us to benchmark next. Contributions are more than welcome! ♥️
+
+## Benchmark datasets
+
+Evaluating Speech Recognition systems is a hard problem. We use the multi-dataset benchmarking strategy proposed in the 
+[ESB paper](https://arxiv.org/abs/2210.13352) to obtain robust evaluation scores for each model.
+
+ESB is a benchmark for evaluating the performance of a single automatic speech recognition (ASR) system across a broad 
+set of speech datasets. It comprises eight English speech recognition datasets, capturing a broad range of domains, 
+acoustic conditions, speaker styles, and transcription requirements. As such, it gives a better indication of how 
+a model is likely to perform on downstream ASR compared to evaluating it on one dataset alone.
+
+The ESB score is calculated as a macro-average of the WER scores across the ESB datasets. The models in the leaderboard
+are ranked based on their average WER scores, from lowest to highest.
+
+| Dataset                                                                                 | Domain                      | Speaking Style        | Train (h) | Dev (h) | Test (h) | Transcriptions     | License         |
+|-----------------------------------------------------------------------------------------|-----------------------------|-----------------------|-----------|---------|----------|--------------------|-----------------|
+| [LibriSpeech](https://huggingface.co/datasets/librispeech_asr)                          | Audiobook                   | Narrated              | 960       | 11      | 11       | Normalised         | CC-BY-4.0       |
+| [Common Voice 9](https://huggingface.co/datasets/mozilla-foundation/common_voice_9_0)   | Wikipedia                   | Narrated              | 1409      | 27      | 27       | Punctuated & Cased | CC0-1.0         |
+| [VoxPopuli](https://huggingface.co/datasets/facebook/voxpopuli)                         | European Parliament         | Oratory               | 523       | 5       | 5        | Punctuated         | CC0             |
+| [TED-LIUM](https://huggingface.co/datasets/LIUM/tedlium)                                | TED talks                   | Oratory               | 454       | 2       | 3        | Normalised         | CC-BY-NC-ND 3.0 |
+| [GigaSpeech](https://huggingface.co/datasets/speechcolab/gigaspeech)                    | Audiobook, podcast, YouTube | Narrated, spontaneous | 2500      | 12      | 40       | Punctuated         | apache-2.0      |
+| [SPGISpeech](https://huggingface.co/datasets/kensho/spgispeech)                         | Fincancial meetings         | Oratory, spontaneous  | 4900      | 100     | 100      | Punctuated & Cased | User Agreement  |
+| [Earnings-22](https://huggingface.co/datasets/revdotcom/earnings22)                     | Fincancial meetings         | Oratory, spontaneous  | 105       | 5       | 5        | Punctuated & Cased | CC-BY-SA-4.0    |
+| [AMI](https://huggingface.co/datasets/edinburghcstr/ami)                                | Meetings                    | Spontaneous           | 78        | 9       | 9        | Punctuated & Cased | CC-BY-4.0       |
+
+For more details on the individual datasets and how models are evaluated to give the ESB score, refer to the [ESB paper](https://arxiv.org/abs/2210.13352).
+"""

init.py

@@ -0,0 +1,93 @@
+import os 
+from constants import EVAL_REQUESTS_PATH
+from pathlib import Path
+from huggingface_hub import HfApi, Repository
+
+TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
+QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
+QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
+
+hf_api = HfApi(
+    endpoint="https://huggingface.co", 
+    token=TOKEN_HUB, 
+)
+
+def load_all_info_from_dataset_hub():
+    eval_queue_repo = None
+    results_csv_path = None
+    requested_models = None
+
+    passed = True
+    if TOKEN_HUB is None:
+        passed = False
+    else:
+        print("Pulling evaluation requests and results.")
+
+        eval_queue_repo = Repository(
+            local_dir=QUEUE_PATH,
+            clone_from=QUEUE_REPO,
+            use_auth_token=TOKEN_HUB,
+            repo_type="dataset",
+        )
+        eval_queue_repo.git_pull()
+        
+        # Local directory where dataset repo is cloned + folder with eval requests
+        directory = QUEUE_PATH / EVAL_REQUESTS_PATH
+        requested_models = get_all_requested_models(directory)
+        requested_models = [p.stem for p in requested_models]
+        # Local directory where dataset repo is cloned
+        csv_results = get_csv_with_results(QUEUE_PATH)
+        if csv_results is None:
+            passed = False
+    if not passed:
+        print("No HuggingFace token provided. Skipping evaluation requests and results.")
+
+    return eval_queue_repo, requested_models, csv_results
+
+
+def upload_file(requested_model_name, path_or_fileobj):
+    dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
+    dest_repo_file = str(dest_repo_file)
+    hf_api.upload_file(
+            path_or_fileobj=path_or_fileobj,
+            path_in_repo=str(dest_repo_file),
+            repo_id=QUEUE_REPO,
+            token=TOKEN_HUB,
+            repo_type="dataset",
+            commit_message=f"Add {requested_model_name} to eval queue")
+
+def get_all_requested_models(directory):
+    directory = Path(directory)
+    all_requested_models = list(directory.glob("*.txt"))
+    return all_requested_models
+
+def get_csv_with_results(directory):
+    directory = Path(directory)
+    all_csv_files = list(directory.glob("*.csv"))
+    latest = [f for f in all_csv_files if f.stem.endswith("latest")]
+    if len(latest) != 1:
+        return None
+    return latest[0]
+
+
+
+def is_model_on_hub(model_name, revision="main") -> bool:
+    try:
+        model_name = model_name.replace(" ","")
+        author = model_name.split("/")[0]
+        model_id = model_name.split("/")[1]
+        if len(author) == 0 or len(model_id) == 0:
+            return False, "is not a valid model name. Please use the format `author/model_name`."
+    except Exception as e:
+        return False, "is not a valid model name. Please use the format `author/model_name`."
+
+    try:
+        models = list(hf_api.list_models(author=author, search=model_id))
+        matched = [model_name for m in models if m.modelId == model_name]
+        if len(matched) != 1:
+            return False, "was not found on the hub!"
+        else:
+            return True, None
+    except Exception as e:
+        print(f"Could not get the model from the hub.: {e}")
+        return False, "was not found on hub!"

requirements.txt

@@ -0,0 +1,61 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.2.0
+cmake==3.26.4
+contourpy==1.1.0
+Cython==3.0.0
+datasets==2.13.1
+dill==0.3.6
+filelock==3.12.2
+fonttools==4.40.0
+frozenlist==1.4.0
+fsspec==2023.6.0
+huggingface-hub==0.16.4
+idna==3.4
+Jinja2==3.1.2
+kiwisolver==1.4.4
+lit==16.0.6
+MarkupSafe==2.1.3
+matplotlib==3.7.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+networkx==3.1
+numpy==1.25.2
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+packaging==23.1
+pandas==2.0.3
+Pillow==10.0.0
+pyarrow==12.0.1
+python-dateutil==2.8.2
+pytz==2023.3
+PyYAML==6.0.1
+regex==2023.6.3
+requests==2.31.0
+responses==0.18.0
+safetensors==0.3.1
+six==1.16.0
+sympy==1.12
+tokenizers==0.13.3
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.65.0
+triton==2.0.0
+typing_extensions==4.7.1
+tzdata==2023.3
+urllib3==2.0.4
+xxhash==3.2.0
+yarl==1.9.2

utils_display.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown")
+    rtf = ColumnContent("RTF (1e-3) ⬇️", "number")
+    avg_wer = ColumnContent("Average WER ⬇️", "number")
+    ami_wer = ColumnContent("AMI", "number")
+    e22_wer = ColumnContent("Earnings22", "number")
+    gs_wer = ColumnContent("Gigaspeech", "number")
+    lsc_wer = ColumnContent("LS Clean", "number")
+    lso_wer = ColumnContent("LS Other", "number")
+    ss_wer = ColumnContent("SPGISpeech", "number")
+    tl_wer = ColumnContent("Tedlium", "number")
+    vp_wer = ColumnContent("Voxpopuli", "number")
+    cv_wer = ColumnContent("Common Voice", "number")                       
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"