import json import gradio as gr import pandas as pd from huggingface_hub import HfFileSystem RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results" fs = HfFileSystem() def fetch_result_paths(): paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json") # results = [file[len(RESULTS_DATASET_ID) +1:] for file in files] return paths def filter_latest_result_path_per_model(paths): from collections import defaultdict d = defaultdict(list) for path in paths: model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1) d[model_id].append(path) return {model_id: max(paths) for model_id, paths in d.items()} def get_result_path_from_model(model_id, result_path_per_model): return result_path_per_model[model_id] def load_result(result_path) -> pd.DataFrame: with fs.open(result_path, "r") as f: data = json.load(f) model_name = data.get("model_name", "Model") df = pd.json_normalize([data]) return df.iloc[0].rename_axis("Parameters").rename(model_name).to_frame() # .reset_index() def render_result_1(model_id, results): result_path = get_result_path_from_model(model_id, latest_result_path_per_model) result = load_result(result_path) return pd.concat([result, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1).reset_index() def render_result_2(model_id, results): result_path = get_result_path_from_model(model_id, latest_result_path_per_model) result = load_result(result_path) return pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result], axis=1).reset_index() # if __name__ == "__main__": latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths()) with gr.Blocks(fill_height=True) as demo: gr.HTML("

Compare Results of the 🤗 Open LLM Leaderboard

") gr.HTML("

Select 2 results to load and compare

") with gr.Row(): with gr.Column(): model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results") load_btn_1 = gr.Button("Load") with gr.Column(): model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results") load_btn_2 = gr.Button("Load") with gr.Row(): compared_results = gr.Dataframe( label="Results", headers=["Parameters", "Result-1", "Result-2"], interactive=False, column_widths=["30%", "30%", "30%"], wrap=True ) load_btn_1.click( fn=render_result_1, inputs=[model_id_1, compared_results], outputs=compared_results, ) load_btn_2.click( fn=render_result_2, inputs=[model_id_2, compared_results], outputs=compared_results, ) demo.launch()