File size: 4,382 Bytes
25557b5 05c90f4 5b0b818 05c90f4 25557b5 c660995 25557b5 05c90f4 c660995 25557b5 05c90f4 1582397 05c90f4 045a182 1582397 05c90f4 045a182 9e63bd6 e29ab28 25557b5 1582397 e29ab28 045a182 e29ab28 1582397 e29ab28 05c90f4 25557b5 e611814 05c90f4 25557b5 e611814 05c90f4 e611814 05c90f4 e611814 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
import json
import gradio as gr
import pandas as pd
from huggingface_hub import HfFileSystem
RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results"
EXCLUDED_KEYS = {
"pretty_env_info",
"chat_template",
"group_subtasks",
}
EXCLUDED_RESULTS_KEYS = {
"leaderboard",
}
EXCLUDED_RESULTS_LEADERBOARDS_KEYS = {
"alias",
}
fs = HfFileSystem()
def fetch_result_paths():
paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json")
return paths
def filter_latest_result_path_per_model(paths):
from collections import defaultdict
d = defaultdict(list)
for path in paths:
model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1)
d[model_id].append(path)
return {model_id: max(paths) for model_id, paths in d.items()}
def get_result_path_from_model(model_id, result_path_per_model):
return result_path_per_model[model_id]
def load_data(result_path) -> pd.DataFrame:
with fs.open(result_path, "r") as f:
data = json.load(f)
return data
def load_result(model_id):
result_path = get_result_path_from_model(model_id, latest_result_path_per_model)
data = load_data(result_path)
df = to_dataframe(data)
result = [
to_vertical(df),
to_vertical(filter_results(df))
]
return result
def to_vertical(df):
df = df.T.rename_axis("Parameters")
df.index = df.index.str.join(".")
return df
def to_dataframe(data):
df = pd.json_normalize([{key: value for key, value in data.items() if key not in EXCLUDED_KEYS}])
# df.columns = df.columns.str.split(".") # .split return a list instead of a tuple
df.columns = list(map(lambda x: tuple(x.split(".")), df.columns))
df.index = [data.get("model_name", "Model")]
return df
def filter_results(df):
df = df.loc[:, df.columns.str[0] == "results"]
df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)]
df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)]
return df
def concat_result_1(result_1, results):
return pd.concat([result_1, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1).reset_index()
def concat_result_2(result_2, results):
return pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result_2], axis=1).reset_index()
def render_result_1(model_id, *results):
result = load_result(model_id)
return [concat_result_1(*result_args) for result_args in zip(result, results)]
def render_result_2(model_id, *results):
result = load_result(model_id)
return [concat_result_2(*result_args) for result_args in zip(result, results)]
# if __name__ == "__main__":
latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths())
with gr.Blocks(fill_height=True) as demo:
gr.HTML("<h1 style='text-align: center;'>Compare Results of the 🤗 Open LLM Leaderboard</h1>")
gr.HTML("<h3 style='text-align: center;'>Select 2 results to load and compare</h3>")
with gr.Row():
with gr.Column():
model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_1 = gr.Button("Load")
with gr.Column():
model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_2 = gr.Button("Load")
with gr.Row():
with gr.Tab("All"):
compared_results_all = gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
)
with gr.Tab("Results"):
compared_results_results = gr.Dataframe(
label="Results",
headers=["Parameters", "Model-1", "Model-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True,
)
load_btn_1.click(
fn=render_result_1,
inputs=[model_id_1, compared_results_all, compared_results_results],
outputs=[compared_results_all, compared_results_results],
)
load_btn_2.click(
fn=render_result_2,
inputs=[model_id_2, compared_results_all, compared_results_results],
outputs=[compared_results_all, compared_results_results],
)
demo.launch()
|