import json import gradio as gr import pandas as pd from huggingface_hub import HfFileSystem RESULTS_DATASET_ID = "datasets/open-llm-leaderboard/results" EXCLUDED_KEYS = { "pretty_env_info", "chat_template", "group_subtasks", } EXCLUDED_RESULTS_KEYS = { "leaderboard", } EXCLUDED_RESULTS_LEADERBOARDS_KEYS = { "alias", } fs = HfFileSystem() def fetch_result_paths(): paths = fs.glob(f"{RESULTS_DATASET_ID}/**/**/*.json") return paths def filter_latest_result_path_per_model(paths): from collections import defaultdict d = defaultdict(list) for path in paths: model_id, _ = path[len(RESULTS_DATASET_ID) +1:].rsplit("/", 1) d[model_id].append(path) return {model_id: max(paths) for model_id, paths in d.items()} def get_result_path_from_model(model_id, result_path_per_model): return result_path_per_model[model_id] def load_data(result_path) -> pd.DataFrame: with fs.open(result_path, "r") as f: data = json.load(f) return data def load_result(model_id): result_path = get_result_path_from_model(model_id, latest_result_path_per_model) data = load_data(result_path) df = to_dataframe(data) result = [ to_vertical(df), to_vertical(filter_results(df)), to_vertical(filter_configs(df)), ] return result def to_vertical(df): df = df.T.rename_axis("Parameters") df.index = df.index.str.join(".") return df def to_dataframe(data): df = pd.json_normalize([{key: value for key, value in data.items() if key not in EXCLUDED_KEYS}]) # df.columns = df.columns.str.split(".") # .split return a list instead of a tuple df.columns = list(map(lambda x: tuple(x.split(".")), df.columns)) df.index = [data.get("model_name", "Model")] return df def filter_results(df): df = df.loc[:, df.columns.str[0] == "results"] df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)] # df.columns.str[1].str = df.columns.str[1].str.removeprefix("leaderboard_") df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)] df.columns = df.columns.str[1:] df.columns = map(lambda x: (x[0].removeprefix("leaderboard_"), *x[1:]), df.columns) return df def filter_configs(df): df = df.loc[:, df.columns.str[0] == "configs"] # df = df.loc[:, ~df.columns.str[1].isin(EXCLUDED_RESULTS_KEYS)] # df = df.loc[:, ~df.columns.str[2].isin(EXCLUDED_RESULTS_LEADERBOARDS_KEYS)] df.columns = df.columns.str[1:] df.columns = map(lambda x: (x[0].removeprefix("leaderboard_"), *x[1:]), df.columns) return df def concat_result_1(result_1, results): return pd.concat([result_1, results.iloc[:, [0, 2]].set_index("Parameters")], axis=1).reset_index() def concat_result_2(result_2, results): return pd.concat([results.iloc[:, [0, 1]].set_index("Parameters"), result_2], axis=1).reset_index() def render_result_1(model_id, *results): result = load_result(model_id) return [concat_result_1(*result_args) for result_args in zip(result, results)] def render_result_2(model_id, *results): result = load_result(model_id) return [concat_result_2(*result_args) for result_args in zip(result, results)] # if __name__ == "__main__": latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths()) with gr.Blocks(fill_height=True) as demo: gr.HTML("

Compare Results of the 🤗 Open LLM Leaderboard

") gr.HTML("

Select 2 results to load and compare

") with gr.Row(): with gr.Column(): model_id_1 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results") load_btn_1 = gr.Button("Load") with gr.Column(): model_id_2 = gr.Dropdown(choices=list(latest_result_path_per_model.keys()), label="Results") load_btn_2 = gr.Button("Load") results = [] with gr.Row(): with gr.Tab("All"): results.append(gr.Dataframe( label="Results", headers=["Parameters", "Model-1", "Model-2"], interactive=False, column_widths=["30%", "30%", "30%"], wrap=True, )) with gr.Tab("Results"): results.append(gr.Dataframe( label="Results", headers=["Parameters", "Model-1", "Model-2"], interactive=False, column_widths=["30%", "30%", "30%"], wrap=True, )) with gr.Tab("Configs"): results.append(gr.Dataframe( label="Results", headers=["Parameters", "Model-1", "Model-2"], interactive=False, column_widths=["30%", "30%", "30%"], wrap=True, )) load_btn_1.click( fn=render_result_1, inputs=[model_id_1, *results], outputs=[*results], ) load_btn_2.click( fn=render_result_2, inputs=[model_id_2, *results], outputs=[*results], ) demo.launch()