"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space.""" import ast import argparse import glob import pickle import gradio as gr import numpy as np import pandas as pd basic_component_values = [None] * 6 leader_component_values = [None] def make_default_md(arena_df, elo_results): total_votes = sum(arena_df["num_battles"]) // 2 total_models = len(arena_df) leaderboard_md = f""" # NeurIPS LLM Merging Competition Leaderboard [Website](https://llm-merging.github.io/index) | [Starter Kit (Github)](https://github.com/llm-merging/LLM-Merging) | [Discord](https://discord.com/invite/dPBHEVnV) | """ return leaderboard_md def make_arena_leaderboard_md(arena_df): total_votes = sum(arena_df["num_battles"]) // 2 total_models = len(arena_df) space = "   " leaderboard_md = f""" Total #models: **{total_models}**.{space} Total #votes: **{"{:,}".format(total_votes)}**.{space} Last updated: June 1, 2024. """ return leaderboard_md def make_category_arena_leaderboard_md(arena_df, arena_subset_df, name="Overall"): total_votes = sum(arena_df["num_battles"]) // 2 total_models = len(arena_df) space = "   " total_subset_votes = sum(arena_subset_df["num_battles"]) // 2 total_subset_models = len(arena_subset_df) leaderboard_md = f"""### {cat_name_to_explanation[name]} #### [Coverage] {space} #models: **{total_subset_models} ({round(total_subset_models/total_models *100)}%)** {space} #votes: **{"{:,}".format(total_subset_votes)} ({round(total_subset_votes/total_votes * 100)}%)**{space} """ return leaderboard_md def make_full_leaderboard_md(elo_results): leaderboard_md = f""" Three benchmarks are displayed: **Test Task 1**, **Test Task 2**, **Test Task 3**. Higher values are better for all benchmarks. """ return leaderboard_md def make_leaderboard_md_live(elo_results): leaderboard_md = f""" # Leaderboard Last updated: {elo_results["last_updated_datetime"]} {elo_results["leaderboard_table"]} """ return leaderboard_md def update_elo_components(max_num_files, elo_results_file): log_files = get_log_files(max_num_files) # Leaderboard if elo_results_file is None: # Do live update battles = clean_battle_data(log_files) elo_results = report_elo_analysis_results(battles) leader_component_values[0] = make_leaderboard_md_live(elo_results) # Basic stats basic_stats = report_basic_stats(log_files) md0 = f"Last updated: {basic_stats['last_updated_datetime']}" md1 = "### Action Histogram\n" md1 += basic_stats["action_hist_md"] + "\n" md2 = "### Anony. Vote Histogram\n" md2 += basic_stats["anony_vote_hist_md"] + "\n" md3 = "### Model Call Histogram\n" md3 += basic_stats["model_hist_md"] + "\n" md4 = "### Model Call (Last 24 Hours)\n" md4 += basic_stats["num_chats_last_24_hours"] + "\n" basic_component_values[0] = md0 basic_component_values[1] = basic_stats["chat_dates_bar"] basic_component_values[2] = md1 basic_component_values[3] = md2 basic_component_values[4] = md3 basic_component_values[5] = md4 def update_worker(max_num_files, interval, elo_results_file): while True: tic = time.time() update_elo_components(max_num_files, elo_results_file) durtaion = time.time() - tic print(f"update duration: {durtaion:.2f} s") time.sleep(max(interval - durtaion, 0)) def load_demo(url_params, request: gr.Request): logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}") return basic_component_values + leader_component_values def model_hyperlink(model_name, link): return f'{model_name}' def load_leaderboard_table_csv(filename, add_hyperlink=True): lines = open(filename).readlines() heads = [v.strip() for v in lines[0].split(",")] rows = [] for i in range(1, len(lines)): row = [v.strip() for v in lines[i].split(",")] for j in range(len(heads)): item = {} for h, v in zip(heads, row): if h == "Arena Elo rating": if v != "-": v = int(ast.literal_eval(v)) else: v = np.nan elif h == "MMLU": if v != "-": v = round(ast.literal_eval(v) * 100, 1) else: v = np.nan elif h == "MT-bench (win rate %)": if v != "-": v = round(ast.literal_eval(v[:-1]), 1) else: v = np.nan elif h == "MT-bench (score)": if v != "-": v = round(ast.literal_eval(v), 2) else: v = np.nan item[h] = v if add_hyperlink: item["Model"] = model_hyperlink(item["Model"], item["Link"]) rows.append(item) return rows def build_basic_stats_tab(): empty = "Loading ..." basic_component_values[:] = [empty, None, empty, empty, empty, empty] md0 = gr.Markdown(empty) gr.Markdown("#### Figure 1: Number of model calls and votes") plot_1 = gr.Plot(show_label=False) with gr.Row(): with gr.Column(): md1 = gr.Markdown(empty) with gr.Column(): md2 = gr.Markdown(empty) with gr.Row(): with gr.Column(): md3 = gr.Markdown(empty) with gr.Column(): md4 = gr.Markdown(empty) return [md0, plot_1, md1, md2, md3, md4] def get_full_table(model_table_df): values = [] for i in range(len(model_table_df)): row = [] model_key = model_table_df.iloc[i]["key"] model_name = model_table_df.iloc[i]["Model"] # model display name row.append(model_name) row.append(np.nan) row.append(np.nan) row.append(np.nan) # row.append(model_table_df.iloc[i]["MT-bench (score)"]) # row.append(model_table_df.iloc[i]["MMLU"]) # Organization row.append(model_table_df.iloc[i]["Organization"]) # license row.append(model_table_df.iloc[i]["License"]) values.append(row) values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9) return values def create_ranking_str(ranking, ranking_difference): if ranking_difference > 0: # return f"{int(ranking)} (\u2191{int(ranking_difference)})" return f"{int(ranking)} \u2191" elif ranking_difference < 0: # return f"{int(ranking)} (\u2193{int(-ranking_difference)})" return f"{int(ranking)} \u2193" else: return f"{int(ranking)}" def recompute_final_ranking(arena_df): # compute ranking based on CI ranking = {} for i, model_a in enumerate(arena_df.index): ranking[model_a] = 1 for j, model_b in enumerate(arena_df.index): if i == j: continue if arena_df.loc[model_b]["rating_q025"] > arena_df.loc[model_a]["rating_q975"]: ranking[model_a] += 1 return list(ranking.values()) def get_arena_table(arena_df, model_table_df, arena_subset_df=None): arena_df = arena_df.sort_values(by=["final_ranking", "rating"], ascending=[True, False]) arena_df["final_ranking"] = recompute_final_ranking(arena_df) arena_df = arena_df.sort_values(by=["final_ranking"], ascending=True) # arena_df["final_ranking"] = range(1, len(arena_df) + 1) # sort by rating if arena_subset_df is not None: # filter out models not in the arena_df arena_subset_df = arena_subset_df[arena_subset_df.index.isin(arena_df.index)] arena_subset_df = arena_subset_df.sort_values(by=["rating"], ascending=False) # arena_subset_df = arena_subset_df.sort_values(by=["final_ranking"], ascending=True) arena_subset_df["final_ranking"] = recompute_final_ranking(arena_subset_df) # keep only the models in the subset in arena_df and recompute final_ranking arena_df = arena_df[arena_df.index.isin(arena_subset_df.index)] # recompute final ranking arena_df["final_ranking"] = recompute_final_ranking(arena_df) # assign ranking by the order arena_subset_df["final_ranking_no_tie"] = range(1, len(arena_subset_df) + 1) arena_df["final_ranking_no_tie"] = range(1, len(arena_df) + 1) # join arena_df and arena_subset_df on index arena_df = arena_subset_df.join(arena_df["final_ranking"], rsuffix="_global", how="inner") arena_df["ranking_difference"] = arena_df["final_ranking_global"] - arena_df["final_ranking"] arena_df = arena_df.sort_values(by=["final_ranking", "rating"], ascending=[True, False]) arena_df["final_ranking"] = arena_df.apply(lambda x: create_ranking_str(x["final_ranking"], x["ranking_difference"]), axis=1) values = [] for i in range(len(arena_df)): row = [] model_key = arena_df.index[i] try: # this is a janky fix for where the model key is not in the model table (model table and arena table dont contain all the same models) model_name = model_table_df[model_table_df["key"] == model_key]["Model"].values[ 0 ] # rank ranking = arena_df.iloc[i].get("final_ranking") or i+1 row.append(ranking) if arena_subset_df is not None: row.append(arena_df.iloc[i].get("ranking_difference") or 0) # model display name row.append(model_name) # elo rating row.append(round(arena_df.iloc[i]["rating"])) # Organization row.append( model_table_df[model_table_df["key"] == model_key]["Organization"].values[0] ) # license row.append( model_table_df[model_table_df["key"] == model_key]["License"].values[0] ) values.append(row) except Exception as e: print(f"{model_key} - {e}") return values key_to_category_name = { "full": "Overall", } cat_name_to_explanation = { "Overall": "Overall Questions", } def build_leaderboard_tab(results_file, leaderboard_table_file, show_plot=False): arena_dfs = {} category_elo_results = {} if results_file is None: # Do live update default_md = "Loading ..." else: with open(results_file, "rb") as fin: elo_results = pickle.load(fin) if "full" in elo_results: print("KEYS ", elo_results.keys()) for k in elo_results.keys(): if k not in key_to_category_name: continue arena_dfs[key_to_category_name[k]] = elo_results[k]["leaderboard_table_df"] category_elo_results[key_to_category_name[k]] = elo_results[k] arena_df = arena_dfs["Overall"] default_md = make_default_md(arena_df, category_elo_results["Overall"]) md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown") if leaderboard_table_file: data = load_leaderboard_table_csv(leaderboard_table_file) model_table_df = pd.DataFrame(data) with gr.Tabs() as tabs: # arena table arena_table_vals = get_full_table(model_table_df) with gr.Tab("Arena Elo", id=0): md = make_arena_leaderboard_md(arena_df) leaderboard_markdown = gr.Markdown(md, elem_id="leaderboard_markdown") with gr.Row(): with gr.Column(scale=2): category_dropdown = gr.Dropdown(choices=list(arena_dfs.keys()), label="Category", value="Overall") default_category_details = make_category_arena_leaderboard_md(arena_df, arena_df, name="Overall") with gr.Column(scale=4, variant="panel"): category_deets = gr.Markdown(default_category_details, elem_id="category_deets") elo_display_df = gr.Dataframe( headers=[ "Rank", "🤖 Model", "⭐ Task 1", "📈 Task 2", "📚 Task 3", "Organization", "License", ], datatype=[ "number", "markdown", "number", "number", "number", "str", "str", ], value=arena_table_vals, elem_id="arena_leaderboard_dataframe", height=700, column_widths=[70, 190, 110, 160, 150, 140], wrap=True, ) gr.Markdown( f"""Note: . """, elem_id="leaderboard_markdown" ) leader_component_values[:] = [default_md] # with gr.Tab("Full Leaderboard", id=0): # md = make_full_leaderboard_md(elo_results) # gr.Markdown(md, elem_id="leaderboard_markdown") # with gr.Row(): # with gr.Column(scale=2): # category_dropdown = gr.Dropdown(choices=list(arena_dfs.keys()), label="Category", value="Overall") # default_category_details = make_category_arena_leaderboard_md(arena_df, arena_df, name="Overall") # with gr.Column(scale=4, variant="panel"): # category_deets = gr.Markdown(default_category_details, elem_id="category_deets") # full_table_vals = get_full_table(model_table_df) # display_df = gr.Dataframe( # headers=[ # "🤖 Model", # "⭐ Task 1", # "📈 Task 2", # "📚 Task 3", # "Organization", # "License", # ], # datatype=["markdown", "number", "number", "number", "str", "str"], # value=full_table_vals, # elem_id="full_leaderboard_dataframe", # column_widths=[200, 100, 100, 100, 150, 150], # height=700, # wrap=True, # ) # gr.Markdown( # f"""Note: . # """, # elem_id="leaderboard_markdown" # ) # leader_component_values[:] = [default_md] if not show_plot: gr.Markdown( """ ## Submit your model [here](). """, elem_id="leaderboard_markdown", ) else: pass def update_leaderboard_df(arena_table_vals): elo_datarame = pd.DataFrame(arena_table_vals, columns=[ "Rank", "🤖 Model", "⭐ Arena Elo", "Organization", "License"]) # goal: color the rows based on the rank with styler def highlight_max(s): # all items in S which contain up arrow should be green, down arrow should be red, otherwise black return ["color: green; font-weight: bold" if "\u2191" in v else "color: red; font-weight: bold" if "\u2193" in v else "" for v in s] def highlight_rank_max(s): return ["color: green; font-weight: bold" if v > 0 else "color: red; font-weight: bold" if v < 0 else "" for v in s] return elo_datarame.style.apply(highlight_max, subset=["Rank"]) def update_leaderboard_and_plots(category): arena_subset_df = arena_dfs[category] arena_subset_df = arena_subset_df[arena_subset_df["num_battles"] > 500] elo_subset_results = category_elo_results[category] arena_df = arena_dfs["Overall"] arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df = arena_subset_df if category != "Overall" else None) if category != "Overall": arena_values = update_leaderboard_df(arena_values) arena_values = gr.Dataframe( headers=[ "Rank", "🤖 Model", "⭐ Arena Elo", "Organization", "License", ], datatype=[ "number", "markdown", "number", "str", "str", ], value=arena_values, elem_id="arena_leaderboard_dataframe", height=700, column_widths=[60, 190, 110, 160, 150, 140], wrap=True, ) else: arena_values = gr.Dataframe( headers=[ "Rank", "🤖 Model", "⭐ Arena Elo", "Organization", "License", ], datatype=[ "number", "markdown", "number", "str", "str", ], value=arena_values, elem_id="arena_leaderboard_dataframe", height=700, column_widths=[70, 190, 110, 160, 150, 140], wrap=True, ) leaderboard_md = make_category_arena_leaderboard_md(arena_df, arena_subset_df, name=category) return arena_values, leaderboard_md category_dropdown.change(update_leaderboard_and_plots, inputs=[category_dropdown], outputs=[display_df, category_deets]) with gr.Accordion( "📝 Citation", open=True, ): citation_md = """ ### Citation Please cite the following paper """ gr.Markdown(citation_md, elem_id="leaderboard_markdown") gr.Markdown(acknowledgment_md) if show_plot: return [md_1] return [md_1] block_css = """ #notice_markdown { font-size: 104% } #notice_markdown th { display: none; } #notice_markdown td { padding-top: 6px; padding-bottom: 6px; } #category_deets { text-align: center; padding: 0px; padding-left: 5px; } #leaderboard_markdown { font-size: 104% } #leaderboard_markdown td { padding-top: 6px; padding-bottom: 6px; } #leaderboard_header_markdown { font-size: 104%; text-align: center; display:block; } #leaderboard_dataframe td { line-height: 0.1em; } #plot-title { text-align: center; display:block; } #non-interactive-button { display: inline-block; padding: 10px 10px; background-color: #f7f7f7; /* Super light grey background */ text-align: center; font-size: 26px; /* Larger text */ border-radius: 0; /* Straight edges, no border radius */ border: 0px solid #dcdcdc; /* A light grey border to match the background */ user-select: none; /* The text inside the button is not selectable */ pointer-events: none; /* The button is non-interactive */ } footer { display:none !important } .sponsor-image-about img { margin: 0 20px; margin-top: 20px; height: 40px; max-height: 100%; width: auto; float: left; } """ acknowledgment_md = """ ### Acknowledgment We thank []() for their generous [sponsorship](). """ def build_demo(elo_results_file, leaderboard_table_file): text_size = gr.themes.sizes.text_lg theme = gr.themes.Base(text_size=text_size) theme.set(button_secondary_background_fill_hover="*primary_300", button_secondary_background_fill_hover_dark="*primary_700") with gr.Blocks( title="LLM Merging Leaderboard", theme=theme, # theme = gr.themes.Base.load("theme.json"), # uncomment to use new cool theme css=block_css, ) as demo: leader_components = build_leaderboard_tab( elo_results_file, leaderboard_table_file, show_plot=True ) return demo if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--share", action="store_true") parser.add_argument("--host", default="0.0.0.0") parser.add_argument("--port", type=int, default=7860) args = parser.parse_args() elo_result_files = glob.glob("elo_results_*.pkl") elo_result_files.sort(key=lambda x: int(x[12:-4])) elo_result_file = elo_result_files[-1] leaderboard_table_files = glob.glob("leaderboard_table_*.csv") leaderboard_table_files.sort(key=lambda x: int(x[18:-4])) leaderboard_table_file = leaderboard_table_files[-1] demo = build_demo(elo_result_file, leaderboard_table_file) demo.launch(share=args.share, server_name=args.host, server_port=args.port)