Spaces:
Running
Running
File size: 3,792 Bytes
3932541 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 |
import requests
import pandas as pd
from tqdm.auto import tqdm
import streamlit as st
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
cer_langs = ["ja", "zh-CN", "zh-HK", "zh-TW"]
def make_clickable(model_name):
link = "https://huggingface.co/" + model_name
return f'<a target="_blank" href="{link}">{model_name}</a>'
def get_model_ids():
api = HfApi()
models = api.list_models(filter="robust-speech-event")
model_ids = [x.modelId for x in models]
return model_ids
def get_metadata(model_id):
try:
readme_path = hf_hub_download(model_id, filename="README.md")
return metadata_load(readme_path)
except requests.exceptions.HTTPError:
# 404 README.md not found
return None
def parse_metric_value(value):
if isinstance(value, str):
"".join(value.split("%"))
try:
value = float(value)
except: # noqa: E722
value = None
elif isinstance(value, float) and value < 1.0:
# assuming that WER is given in 0.xx format
value = 100 * value
elif isinstance(value, list):
if len(value) > 0:
value = value[0]
else:
value = None
value = round(value, 2) if value is not None else None
return value
def parse_metrics_row(meta):
if "model-index" not in meta or "language" not in meta:
return None
lang = meta["language"]
lang = lang[0] if isinstance(lang, list) else lang
for result in meta["model-index"][0]["results"]:
if "dataset" not in result or "metrics" not in result:
continue
dataset = result["dataset"]["type"]
if "args" not in result["dataset"]:
continue
dataset_config = result["dataset"]["args"]
row = {"dataset": dataset, "lang": lang}
for metric in result["metrics"]:
type = metric["type"].lower().strip()
if type not in ["wer", "cer"]:
continue
value = parse_metric_value(metric["value"])
if value is None:
continue
if type not in row or value < row[type]:
# overwrite the metric if the new value is lower (e.g. with LM)
row[type] = value
if "wer" in row or "cer" in row:
return row
return None
@st.cache(ttl=600)
def get_data():
data = []
model_ids = get_model_ids()
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
row = parse_metrics_row(meta)
if row is None:
continue
row["model_id"] = model_id
data.append(row)
return pd.DataFrame.from_records(data)
dataframe = get_data()
dataframe = dataframe.fillna("")
dataframe["model_id"] = dataframe["model_id"].apply(make_clickable)
_, col_center = st.columns([3, 6])
with col_center:
st.image("logo.png", width=200)
st.markdown("# Speech Models Leaderboard")
lang = st.selectbox(
"Language",
sorted(dataframe["lang"].unique()),
index=0,
)
lang_df = dataframe[dataframe.lang == lang]
dataset = st.selectbox(
"Dataset",
sorted(lang_df["dataset"].unique()),
index=0,
)
dataset_df = lang_df[lang_df.dataset == dataset]
if lang in cer_langs:
dataset_df = dataset_df[["model_id", "cer"]]
dataset_df.sort_values("cer", inplace=True)
else:
dataset_df = dataset_df[["model_id", "wer"]]
dataset_df.sort_values("wer", inplace=True)
dataset_df.rename(
columns={
"model_id": "Model",
"wer": "WER (lower is better)",
"cer": "CER (lower is better)",
},
inplace=True,
)
st.write(dataset_df.to_html(escape=False, index=None), unsafe_allow_html=True)
|