File size: 2,995 Bytes
25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 c660995 25557b5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
import json
import gradio as gr
import pandas as pd
from huggingface_hub import HfFileSystem
RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results"
fs = HfFileSystem()
def fetch_result_paths():
paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json")
# results = [file[len(RESULTS_DATASET_ID) +1:] for file in files]
return paths
def filter_latest_result_path_per_model(paths):
from collections import defaultdict
d = defaultdict(list)
for path in paths:
model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1)
d[model_id].append(path)
return {model_id: max(paths) for model_id, paths in d.items()}
def get_result_path_from_model(model_id, result_path_per_model):
return result_path_per_model[model_id]
def load_result(result_path) -> pd.DataFrame:
with fs.open(result_path, "r") as f:
data = json.load(f)
model_name = data.get("model_name", "Model")
df = pd.json_normalize([data])
return df.iloc[0].rename_axis("Parameters").rename(model_name).to_frame() # .reset_index()
def render_result_1(model_id, results):
result_path = get_result_path_from_model(model_id, latest_result_path_per_model)
result = load_result(result_path)
return pd.concat([result, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1).reset_index()
def render_result_2(model_id, results):
result_path = get_result_path_from_model(model_id, latest_result_path_per_model)
result = load_result(result_path)
return pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result], axis=1).reset_index()
if __name__ == "__main__":
latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths())
with gr.Blocks(fill_height=True) as demo:
gr.HTML("<h1 style='text-align: center;'>Compare Results of the 🤗 Open LLM Leaderboard</h1>")
gr.HTML("<h3 style='text-align: center;'>Select 2 results to load and compare</h3>")
with gr.Row():
with gr.Column():
model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_1 = gr.Button("Load")
with gr.Column():
model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results")
load_btn_2 = gr.Button("Load")
with gr.Row():
compared_results = gr.Dataframe(
label="Results",
headers=["Parameters", "Result-1", "Result-2"],
interactive=False,
column_widths=["30%", "30%", "30%"],
wrap=True
)
load_btn_1.click(
fn=render_result_1,
inputs=[model_id_1, compared_results],
outputs=compared_results,
)
load_btn_2.click(
fn=render_result_2,
inputs=[model_id_2, compared_results],
outputs=compared_results,
)
demo.launch()
|