comparator / app.py
albertvillanova's picture
Add Configs tab
fb9885c verified
raw
history blame
4.81 kB
import json
import gradio as gr
import pandas as pd
from huggingface_hub import HfFileSystem
RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results"
EXCLUDED_KEYS = {
"pretty_env_info",
"chat_template",
"group_subtasks",
}
EXCLUDED_RESULTS_KEYS = {
"leaderboard",
}
EXCLUDED_RESULTS_LEADERBOARDS_KEYS = {
"alias",
}
fs = HfFileSystem()
def fetch_result_paths():
paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json")
return paths
def filter_latest_result_path_per_model(paths):
from collections import defaultdict
d = defaultdict(list)
for path in paths:
model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1)
d[model_id].append(path)
return {model_id: max(paths) for model_id, paths in d.items()}
def get_result_path_from_model(model_id, result_path_per_model):
return result_path_per_model[model_id]
def load_data(result_path) -> pd.DataFrame:
with fs.open(result_path, "r") as f:
data = json.load(f)
return data
def load_result(model_id):
result_path = get_result_path_from_model(model_id, latest_result_path_per_model)
data = load_data(result_path)
df = to_dataframe(data)
result = [
to_vertical(df),
to_vertical(filter_results(df)),
to_vertical(filter_configs(df)),
]
return result
def to_vertical(df):
df = df.T.rename_axis("Parameters")
df.index = df.index.str.join(".")
return df
def to_dataframe(data):
df = pd.json_normalize([{key: value for key, value in data.items() if key not in EXCLUDED_KEYS}])
# df.columns = df.columns.str.split(".") # .split return a list instead of a tuple
df.columns = list(map(lambda x: tuple(x.split(".")), df.columns))
df.index = [data.get("model_name", "Model")]
return df
def filter_results(df):
df = df.loc[:, df.columns.str[0] == "results"]
df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)]
df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)]
return df
def filter_configs(df):
df = df.loc[:, df.columns.str[0] == "configs"]
# df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)]
# df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)]
return df
def concat_result_1(result_1, results):
return pd.concat([result_1, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1).reset_index()
def concat_result_2(result_2, results):
return pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result_2], axis=1).reset_index()
def render_result_1(model_id, *results):
result = load_result(model_id)
return [concat_result_1(*result_args) for result_args in zip(result, results)]
def render_result_2(model_id, *results):
result = load_result(model_id)
return [concat_result_2(*result_args) for result_args in zip(result, results)]
# if __name__ == "__main__":
latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths())
with gr.Blocks(fill_height=True) as demo:
gr.HTML("<h1 style='text-align: center;'>Compare Results of the πŸ€— Open LLM Leaderboard</h1>")
gr.HTML("<h3 style='text-align: center;'>Select 2 results to load and compare</h3>")
with gr.Row():
with gr.Column():
model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_1 = gr.Button("Load")
with gr.Column():
model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_2 = gr.Button("Load")
results = []
with gr.Row():
with gr.Tab("All"):
results.append(gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
))
with gr.Tab("Results"):
results.append(gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
))
with gr.Tab("Configs"):
results.append(gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
))
load_btn_1.click(
fn=render_result_1,
inputs=[model_id_1, *results],
outputs=[*results],
)
load_btn_2.click(
fn=render_result_2,
inputs=[model_id_2, *results],
outputs=[*results],
)
demo.launch()