Modify app.py to read and display the new multilingual results

app.py

@@ -9,30 +9,32 @@ import numpy as np
 import pandas as pd
 
 
-# notebook_url = "https://colab.research.google.com/drive/1KdwokPjirkTmpO_P1WByFNFiqxWQquwH#scrollTo=o_CpbkGEbhrK
+original_notebook_url = "https://colab.research.google.com/drive/1KdwokPjirkTmpO_P1WByFNFiqxWQquwH#scrollTo=o_CpbkGEbhrK"
 notebook_url = "https://colab.research.google.com/drive/11eWOT3VAAWRRrs1CSsAg84hIaJvH2ThK?usp=sharing"
-
+data_link = "https://drive.google.com/file/d/1_72443egRzwRTmJfIyOQcf1ug7sKbqbX/view?usp=sharing"
+original_leaderboard_link = "{original_leaderboard_link}"
 
 basic_component_values = [None] * 6
 leader_component_values = [None] * 5
 
 
-def make_default_md(arena_df, elo_results):
-    total_votes = sum(arena_df["num_battles"]) // 2
-    total_models = len(arena_df)
-
+def make_default_md(languages_names):
     leaderboard_md = f"""
-# 🏆 LMSYS Chatbot Arena Leaderboard
-| [Vote](https://chat.lmsys.org) | [Blog](https://lmsys.org/blog/2023-05-03-arena/) | [GitHub](https://github.com/lm-sys/FastChat) | [Paper](https://arxiv.org/abs/2306.05685) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/HSWAKCrnFx) |
+# 🏆 Multilingual LMSYS Chatbot Arena Leaderboard
+LMSYS Org link's: | [Vote](https://chat.lmsys.org) | [Blog](https://lmsys.org/blog/2023-05-03-arena/) | [GitHub](https://github.com/lm-sys/FastChat) | [Paper](https://arxiv.org/abs/2306.05685) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/HSWAKCrnFx) |
 
 LMSYS [Chatbot Arena](https://lmsys.org/blog/2023-05-03-arena/) is a crowdsourced open platform for LLM evals.
-We've collected over **500,000** human preference votes to rank LLMs with the Elo ranking system.
+They've collected over **500,000** human preference votes to rank LLMs with the Elo ranking system.
+
+This leaderboard is a fork derived from the [🏆LMSYS Chatbot Arena Leaderboard]({original_leaderboard_link}). The LMSYS Org provides [data]({original_notebook_url}) that contains the language inferred for each conversation using the polyglot package, we use this data for featuring additional metrics and analysis for each individual language, with a particular emphasis on non-English languages.  
+
+In the "By Language" section, we offer individual metrics for the following languages: {", ".join(languages_names[:-1])}, and {languages_names[-1]}.
 """
     return leaderboard_md
 
 
 def make_arena_leaderboard_md(arena_df):
-    total_votes = sum(arena_df["num_battles"]) // 2
+    total_votes = int(sum(arena_df["num_battles"]) // 2)
     total_models = len(arena_df)
 
     leaderboard_md = f"""
@@ -46,8 +48,8 @@ Contribute your vote 🗳️ at [chat.lmsys.org](https://chat.lmsys.org)! Find m
 def make_full_leaderboard_md(elo_results):
     leaderboard_md = f"""
 Three benchmarks are displayed: **Arena Elo**, **MT-Bench** and **MMLU**.
-- [Chatbot Arena](https://chat.lmsys.org/?arena) - a crowdsourced, randomized battle platform. We use 500K+ user votes to compute Elo ratings.
-- [MT-Bench](https://arxiv.org/abs/2306.05685): a set of challenging multi-turn questions. We use GPT-4 to grade the model responses.
+- [Chatbot Arena](https://chat.lmsys.org/?arena) - a crowdsourced, randomized battle platform. They use 500K+ user votes to compute Elo ratings.
+- [MT-Bench](https://arxiv.org/abs/2306.05685): a set of challenging multi-turn questions. They use GPT-4 to grade the model responses.
 - [MMLU](https://arxiv.org/abs/2009.03300) (5-shot): a test to measure a model's multitask accuracy on 57 tasks.
 
 💻 Code: The MT-bench scores (single-answer grading on a scale of 10) are computed by [fastchat.llm_judge](https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge).
@@ -220,6 +222,8 @@ def get_arena_table(arena_df, model_table_df):
         # model display name
         row.append(model_name)
         # elo rating
+        if pd.isna(arena_df.iloc[i]["rating"]):
+            continue
         row.append(round(arena_df.iloc[i]["rating"]))
         upper_diff = round(
             arena_df.iloc[i]["rating_q975"] - arena_df.iloc[i]["rating"]
@@ -228,6 +232,8 @@ def get_arena_table(arena_df, model_table_df):
             arena_df.iloc[i]["rating"] - arena_df.iloc[i]["rating_q025"]
         )
         row.append(f"+{upper_diff}/-{lower_diff}")
+        # Avg. Win Rate
+        row.append(f'{round(arena_df.iloc[i]["avg_win_rate_no_tie"] * 100, 1):04.1f}%')
         # num battles
         row.append(round(arena_df.iloc[i]["num_battles"]))
         # Organization
@@ -239,139 +245,159 @@ def get_arena_table(arena_df, model_table_df):
             model_table_df[model_table_df["key"] == model_key]["License"].values[0]
         )
 
-        cutoff_date = model_table_df[model_table_df["key"] == model_key]["Knowledge cutoff date"].values[0]
-        if cutoff_date == "-":
-            row.append("Unknown")
-        else:
-            row.append(cutoff_date)
+        #cutoff_date = model_table_df[model_table_df["key"] == model_key]["Knowledge cutoff date"].values[0]
+        #if cutoff_date == "-":
+        #    row.append("Unknown")
+        #else:
+        #    row.append(cutoff_date)
+
         values.append(row)
     return values
 
-def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=False):
-    if elo_results_file is None:  # Do live update
-        default_md = "Loading ..."
-        p1 = p2 = p3 = p4 = None
-    else:
-        with open(elo_results_file, "rb") as fin:
-            elo_results = pickle.load(fin)
-            if "full" in elo_results:
-                elo_results = elo_results["full"]
-
-        p1 = elo_results["win_fraction_heatmap"]
-        p2 = elo_results["battle_count_heatmap"]
-        p3 = elo_results["bootstrap_elo_rating"]
-        p4 = elo_results["average_win_rate_bar"]
-        arena_df = elo_results["leaderboard_table_df"]
-        default_md = make_default_md(arena_df, elo_results)
-
-    md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
-    if leaderboard_table_file:
-        data = load_leaderboard_table_csv(leaderboard_table_file)
-        model_table_df = pd.DataFrame(data)
-
-        with gr.Tabs() as tabs:
-            # arena table
-            arena_table_vals = get_arena_table(arena_df, model_table_df)
-            with gr.Tab("Arena Elo", id=0):
-                md = make_arena_leaderboard_md(arena_df)
-                gr.Markdown(md, elem_id="leaderboard_markdown")
-                gr.Dataframe(
-                    headers=[
-                        "Rank",
-                        "🤖 Model",
-                        "⭐ Arena Elo",
-                        "📊 95% CI",
-                        "🗳️ Votes",
-                        "Organization",
-                        "License",
-                        "Knowledge Cutoff",
-                    ],
-                    datatype=[
-                        "str",
-                        "markdown",
-                        "number",
-                        "str",
-                        "number",
-                        "str",
-                        "str",
-                        "str",
-                    ],
-                    value=arena_table_vals,
-                    elem_id="arena_leaderboard_dataframe",
-                    height=700,
-                    column_widths=[50, 200, 120, 100, 100, 150, 150, 100],
-                    wrap=True,
-                )
-            with gr.Tab("Full Leaderboard", id=1):
-                md = make_full_leaderboard_md(elo_results)
-                gr.Markdown(md, elem_id="leaderboard_markdown")
-                full_table_vals = get_full_table(arena_df, model_table_df)
-                gr.Dataframe(
-                    headers=[
-                        "🤖 Model",
-                        "⭐ Arena Elo",
-                        "📈 MT-bench",
-                        "📚 MMLU",
-                        "Organization",
-                        "License",
-                    ],
-                    datatype=["markdown", "number", "number", "number", "str", "str"],
-                    value=full_table_vals,
-                    elem_id="full_leaderboard_dataframe",
-                    column_widths=[200, 100, 100, 100, 150, 150],
-                    height=700,
-                    wrap=True,
-                )
-        if not show_plot:
-            gr.Markdown(
-                """ ## Visit our [HF space](https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard) for more analysis!
-                If you want to see more models, please help us [add them](https://github.com/lm-sys/FastChat/blob/main/docs/arena.md#how-to-add-a-new-model).
-                """,
-                elem_id="leaderboard_markdown",
-            )
-    else:
-        pass
-
-    gr.Markdown(
-        f"""Note: we take the 95% confidence interval into account when determining a model's ranking.
-A model is ranked higher only if its lower bound of model score is higher than the upper bound of the other model's score.
-See Figure 3 below for visualization of the confidence intervals.
-""",
-        elem_id="leaderboard_markdown"
+def create_leaderboard_from_results(elo_results, model_table_df, show_plot, show_language_plot=False):
+    p0 = elo_results["inferred_languages_bar"]
+    p1 = elo_results["win_fraction_heatmap"]
+    p2 = elo_results["battle_count_heatmap"]
+    p3 = elo_results["bootstrap_elo_rating"]
+    p4 = elo_results["average_win_rate_bar"]
+    arena_df = elo_results["leaderboard_table_df"]
+    arena_table_vals = get_arena_table(arena_df, model_table_df)
+    
+    md = make_arena_leaderboard_md(arena_df)
+    gr.Markdown(md, elem_id="leaderboard_markdown")
+    gr.Dataframe(
+        headers=[
+            "Rank",
+            "🤖 Model",
+            "⭐ Arena Elo",
+            "📊 95% CI",
+            "🏆 Avg. Win Rate",
+            "🗳️ Votes",
+            "Organization",
+            "License",
+            #"Knowledge Cutoff",
+        ],
+        datatype=[
+            "str",
+            "markdown",
+            "number",
+            "str",
+            "str",
+            "number",
+            "str",
+            "str",
+            #"str",
+        ],
+        value=arena_table_vals,
+        elem_id="arena_leaderboard_dataframe",
+        height=700,
+        column_widths=[50, 200, 120, 100, 150, 100, 125, 125],#, 100],
+        wrap=True,
     )
 
-    leader_component_values[:] = [default_md, p1, p2, p3, p4]
+    gr.Markdown(
+            f"""Note¹: we take the 95% confidence interval into account when determining a model's ranking.
+    A model is ranked higher only if its lower bound of model score is higher than the upper bound of the other model's score. See Figure {3+int(show_language_plot)} below for visualization of the confidence intervals.  
+    Note²: The Average Win Rate is calculated by assuming uniform sampling and no ties.
+    """,
+            elem_id="leaderboard_markdown"
+        )
 
-    if show_plot:
+    if not show_plot:
         gr.Markdown(
-            f"""## More Statistics for Chatbot Arena\n
-Below are figures for more statistics. The code for generating them is also included in this [notebook]({notebook_url}).
-You can find more discussions in this blog [post](https://lmsys.org/blog/2023-12-07-leaderboard/).
-    """,
+            f""" ## Visit our [HF space]({original_leaderboard_link}) for more analysis!
+            If you want to see more models, please help us [add them](https://github.com/lm-sys/FastChat/blob/main/docs/arena.md#how-to-add-a-new-model).
+            """,
+            elem_id="leaderboard_markdown",
+        )
+    else:
+        gr.Markdown(
+                    f"""## More Statistics for Chatbot Arena\n
+        Below are figures for more statistics. The code for generating them is also included in this [notebook]({notebook_url}).
+        You can find more discussions in this blog [post](https://lmsys.org/blog/2023-12-07-leaderboard/).
+            """,
             elem_id="leaderboard_markdown"
         )
+        fig_id = 1
+        if show_language_plot:
+            gr.Markdown(
+                f"#### Figure {fig_id}: Battle counts for the Top 15 Languages"
+            )
+            plot_0 = gr.Plot(p0, show_label=False)
+            fig_id += 1
         with gr.Row():
             with gr.Column():
                 gr.Markdown(
-                    "#### Figure 1: Fraction of Model A Wins for All Non-tied A vs. B Battles"
+                    f"#### Figure {fig_id}: Fraction of Model A Wins for All Non-tied A vs. B Battles"
                 )
                 plot_1 = gr.Plot(p1, show_label=False)
+                fig_id += 1
             with gr.Column():
                 gr.Markdown(
-                    "#### Figure 2: Battle Count for Each Combination of Models (without Ties)"
+                    f"#### Figure {fig_id}: Battle Count for Each Combination of Models (without Ties)"
                 )
                 plot_2 = gr.Plot(p2, show_label=False)
+                fig_id += 1
         with gr.Row():
             with gr.Column():
                 gr.Markdown(
-                    "#### Figure 3: Confidence Intervals on Model Strength (via Bootstrapping)"
+                    f"#### Figure {fig_id}: Confidence Intervals on Model Strength (via Bootstrapping)"
                 )
                 plot_3 = gr.Plot(p3, show_label=False)
+                fig_id += 1
             with gr.Column():
                 gr.Markdown(
-                    "#### Figure 4: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)"
+                    f"#### Figure {fig_id}: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)"
                 )
                 plot_4 = gr.Plot(p4, show_label=False)
+                fig_id += 1
+
+    return p1, p2, p3, p4, plot_1, plot_2, plot_3, plot_4
+
+def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=False):
+    if elo_results_file is None:  # Do live update
+        default_md = "Loading ..."
+        p1 = p2 = p3 = p4 = None
+    else:
+        with open(elo_results_file, "rb") as fin:
+            elo_results = pickle.load(fin)
+            #if "non-english" in elo_results:
+            #    elo_results = elo_results["non-english"]
+
+    languages = [lang for lang in elo_results if lang not in ["non-english", "full"]]
+    languages = languages[::-1][:-3]
+    languages_names = [lang[0].upper() + lang[1:] for lang in languages]
+
+    default_md = make_default_md(languages_names)
+    md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
+
+    if leaderboard_table_file:
+        data = load_leaderboard_table_csv(leaderboard_table_file)
+        model_table_df = pd.DataFrame(data)
+
+        with gr.Tabs() as tabs:
+            # arena table
+            with gr.Tab("Multilingual (Non-English)", id=0):
+                gr.Markdown("This section includes metrics for all interactions that are not in English. See Figure 1 below for the distribution of evaluated languages.")
+                p1, p2, p3, p4, plot_1, plot_2, plot_3, plot_4 = create_leaderboard_from_results(elo_results["non-english"], model_table_df, show_plot, show_language_plot=True)
+            with gr.Tab("Multilingual (All langs)", id=1):
+                gr.Markdown(f"This section includes metrics for all interactions, should be the same as the original [🏆LMSYS Chatbot Arena Leaderboard]({original_leaderboard_link}). See Figure 1 below for the distribution of evaluated languages.")
+                create_leaderboard_from_results(elo_results['full'], model_table_df, show_plot, show_language_plot=True)
+            with gr.Tab("By Language", id=2):
+                with gr.Tabs() as tabs:
+                    for i, lang in enumerate(languages):
+                        elo_result = elo_results[lang]
+                        lang = lang[0].upper() + lang[1:]
+                        arena_df = elo_result['leaderboard_table_df']
+                        size = round((sum(arena_df['num_battles']) // 2) / 1000)
+                        with gr.Tab(lang + f" ({size}K)", id=i+3):
+                            gr.Markdown(f"This section includes metrics for all interactions that are in {lang}.")
+                            create_leaderboard_from_results(elo_result, model_table_df, show_plot)
+            
+    else:
+        pass
+
+    leader_component_values[:] = [default_md, p1, p2, p3, p4]
 
     with gr.Accordion(
         "📝 Citation",
@@ -379,7 +405,7 @@ You can find more discussions in this blog [post](https://lmsys.org/blog/2023-12
     ):
         citation_md = """
         ### Citation
-        Please cite the following paper if you find our leaderboard or dataset helpful.
+        Please cite the following paper if you find the leaderboard or dataset helpful.
         ```
         @misc{chiang2024chatbot,
             title={Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference},
@@ -431,20 +457,12 @@ footer {
 }
 """
 
-acknowledgment_md = """
+acknowledgment_md = f"""
 ### Acknowledgment
-We thank [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).
-
-<div class="sponsor-image-about">
-    <img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
-    <img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
-    <img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
-    <img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
-    <img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
-    <img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
-</div>
+Thanks to LMSYS team for providing the open-source [data]({original_notebook_url}) and the original [🏆LMSYS Chatbot Arena Leaderboard]({original_leaderboard_link}).
 """
 
+'''
 def build_demo(elo_results_file, leaderboard_table_file):
     text_size = gr.themes.sizes.text_lg
 
@@ -457,20 +475,26 @@ def build_demo(elo_results_file, leaderboard_table_file):
             elo_results_file, leaderboard_table_file, show_plot=True
         )
     return demo
+'''
 
+elo_result_files = glob.glob("elo_results_*.pkl")
+elo_result_files.sort(key=lambda x: int(x[12:-4]))
+elo_result_file = elo_result_files[-1]
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--share", action="store_true")
-    args = parser.parse_args()
+leaderboard_table_files = glob.glob("leaderboard_table_*.csv")
+leaderboard_table_files.sort(key=lambda x: int(x[18:-4]))
+leaderboard_table_file = leaderboard_table_files[-1]
 
-    elo_result_files = glob.glob("elo_results_*.pkl")
-    elo_result_files.sort(key=lambda x: int(x[12:-4]))
-    elo_result_file = elo_result_files[-1]
+text_size = gr.themes.sizes.text_lg
 
-    leaderboard_table_files = glob.glob("leaderboard_table_*.csv")
-    leaderboard_table_files.sort(key=lambda x: int(x[18:-4]))
-    leaderboard_table_file = leaderboard_table_files[-1]
+with gr.Blocks(
+    title="Chatbot Arena Leaderboard",
+    theme=gr.themes.Base(text_size=text_size),
+    css=block_css,
+) as demo:
+    leader_components = build_leaderboard_tab(
+        elo_result_file, leaderboard_table_file, show_plot=True
+    )
 
-    demo = build_demo(elo_result_file, leaderboard_table_file)
-    demo.launch(share=args.share)
+if __name__ == "__main__":
+    demo.launch()