Track large files with Git LFS, and expand app to include a data explorer and more length-based visualizations.

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/df_response_judging.jsonl filter=lfs diff=lfs merge=lfs -text
+data/df_responses.jsonl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,2 +1,3 @@
 env/
-submodules/
+submodules/
+.DS_Store

app.py

@@ -3,6 +3,7 @@ import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 import statsmodels.api as sm
+import random
 
 # Set the layout to wide
 st.set_page_config(layout="wide")
@@ -36,206 +37,615 @@ def prep_rankings_table(df, y_column):
     return df_copy
 
 
+def get_preference(preference_score):
+    rounded_preference_score = int(preference_score.round(0).iloc[0])
+    return get_preference_from_rounded_score(rounded_preference_score)
+    # if rounded_preference_score == 2:
+    #     return "[2>1]"
+    # elif rounded_preference_score == 1:
+    #     return "[1>2]"
+
+
+def get_preference_from_rounded_score(score):
+    if score == 2:
+        return "[2>1]"
+    elif score == 1:
+        return "[1>2]"
+    return "[1=2]"
+    # raise ValueError(f"Invalid score: {score}")
+
+
 def app():
+    fixed_model = "gpt4_1106_preview"
+
+    # Ensure to initialize session state variables if they do not exist
+    if "selected_instruction" not in st.session_state:
+        st.session_state.selected_instruction = None
+
+    if "selected_model" not in st.session_state:
+        st.session_state.selected_model = "gpt4"
+
+    if "selected_judge" not in st.session_state:
+        st.session_state.selected_judge = None
+
+    if "selected_dataset" not in st.session_state:
+        st.session_state.selected_dataset = "NEW"
+
+    if "instruction_options" not in st.session_state:
+        st.session_state.instruction_options = []
+
+    # Function to update the instruction options based on selected dataset
+    def update_instruction_options():
+        selected_dataset = st.session_state.dataset_selector
+        if selected_dataset == "all" or selected_dataset == "NEW":
+            instruction_options = df_response_judging["instruction"].unique().tolist()
+        elif (
+            selected_dataset == "None"
+            or selected_dataset is None
+            or str(selected_dataset) == ""
+        ):
+            instruction_options = (
+                df_response_judging[pd.isna(df_response_judging["dataset"])][
+                    "instruction"
+                ]
+                .unique()
+                .tolist()
+            )
+        else:
+            instruction_options = (
+                df_response_judging[df_response_judging["dataset"] == selected_dataset][
+                    "instruction"
+                ]
+                .unique()
+                .tolist()
+            )
+
+        st.session_state.instruction_options = instruction_options
+
+    def update_instruction():
+        st.session_state.selected_instruction = st.session_state.instruction_selector
+
+    def update_model():
+        st.session_state.selected_model = st.session_state.model_selector
+
+    def update_judge():
+        st.session_state.selected_judge = st.session_state.judge_selector
+
+    def randomize_selection():
+        st.session_state.dataset_selector = random.choice(
+            ["all"] + df_response_judging["dataset"].dropna().unique().tolist()
+        )
+        st.session_state.selected_model = random.choice(model_options)
+        update_instruction_options()
+        st.session_state.selected_instruction = random.choice(
+            st.session_state.instruction_options
+        )
+
     st.title("AlpacaEval Visualizations")
 
-    st.markdown("## Win rate vs. overall mean length")
+    outer_tabs = st.tabs(["Length bias in overall win rate", "Data explorer"])
 
     # Load the data
-    df = pd.read_json("data/model_win_rates.json")
-
-    # Add a model name column for hover labels
-    df["model_name"] = df.index.astype(str)
-
-    # Define the preset groups
-    presets = {
-        "gpt": df[df["model_name"].str.contains("openai|gpt", case=False)][
-            "model_name"
-        ].tolist(),
-        "claude": df[df["model_name"].str.contains("claude", case=False)][
-            "model_name"
-        ].tolist(),
-        "moa": df[df["model_name"].str.contains("moa", case=False)][
-            "model_name"
-        ].tolist(),
-        "llama": df[df["model_name"].str.contains("llama", case=False)][
-            "model_name"
-        ].tolist(),
-        "custom": [],
-    }
-
-    # Add radio button for preset groups
-    preset_selection = st.radio(
-        "Select a preset group of models or choose 'custom' to select manually",
-        options=["custom", "gpt", "claude", "moa", "llama"],
+    df = pd.read_json("data/model_win_rates.jsonl", lines=True, orient="records")
+    # df_responses = pd.read_json("data/df_responses.jsonl", lines=True, orient="records")
+    df_response_judging = pd.read_json(
+        "data/df_response_judging.jsonl", lines=True, orient="records"
     )
 
-    # Add multiselect for custom model selection
-    if preset_selection == "custom":
-        selected_models = st.multiselect(
-            "Select models to highlight", options=df["model_name"].unique()
-        )
-    else:
-        selected_models = presets[preset_selection]
-
-    def create_scatter_plot(df, y_column, selected_models, title):
-        fig = go.Figure()
-
-        # Add scatter plots for num_words_mean and num_tokens_mean
-        fig.add_trace(
-            go.Scatter(
-                x=df["num_words_mean"],
-                y=df[y_column],
-                mode="markers",
-                name="words",
-                text=df["model_name"],
-                marker=dict(size=5, color="skyblue"),
-                showlegend=True,
-            )
-        )
-        fig.add_trace(
-            go.Scatter(
-                x=df["num_tokens_mean"],
-                y=df[y_column],
-                mode="markers",
-                name="tokens",
-                text=df["model_name"],
-                marker=dict(size=5, color="orange"),
-                showlegend=True,
-                visible="legendonly",  # Make 'words' trace initially visible only in legend
-            )
-        )
-
-        # Highlight selected models
-        if selected_models:
-            selected_data = df[df["model_name"].isin(selected_models)]
+    # Prepare the model selector options
+    model_options = df_response_judging["generator_2"].unique().tolist()
+
+    with outer_tabs[0]:
+        # Define the preset groups
+        presets = {
+            "gpt": df[df["model_name"].str.contains("openai|gpt", case=False)][
+                "model_name"
+            ].tolist(),
+            "claude": df[df["model_name"].str.contains("claude", case=False)][
+                "model_name"
+            ].tolist(),
+            "moa": df[df["model_name"].str.contains("moa", case=False)][
+                "model_name"
+            ].tolist(),
+            "llama": df[df["model_name"].str.contains("llama", case=False)][
+                "model_name"
+            ].tolist(),
+            "custom": [],
+        }
+
+        # Add radio button for preset groups
+        preset_selection = st.radio(
+            "Select a preset group of models or choose 'custom' to select manually.",
+            options=["custom", "gpt", "claude", "moa", "llama"],
+        )
+
+        st.divider()
+
+        # Add multiselect for custom model selection
+        if preset_selection == "custom":
+            selected_models = st.multiselect(
+                "Select models to highlight", options=df["model_name"].unique()
+            )
+        else:
+            selected_models = presets[preset_selection]
+
+        def create_scatter_plot(df, y_column, selected_models, title):
+            fig = go.Figure()
+
+            # Add scatter plots for num_words_mean and num_tokens_mean
             fig.add_trace(
                 go.Scatter(
-                    x=selected_data["num_words_mean"],
-                    y=selected_data[y_column],
+                    x=df["num_words_mean"],
+                    y=df[y_column],
                     mode="markers",
-                    name="selected words",
-                    text=selected_data["model_name"],
-                    marker=dict(size=10, color="blue"),
+                    name="words",
+                    text=df["model_name"],
+                    marker=dict(size=5, color="skyblue"),
                     showlegend=True,
                 )
             )
             fig.add_trace(
                 go.Scatter(
-                    x=selected_data["num_tokens_mean"],
-                    y=selected_data[y_column],
+                    x=df["num_tokens_mean"],
+                    y=df[y_column],
                     mode="markers",
-                    name="selected tokens",
-                    text=selected_data["model_name"],
-                    marker=dict(size=10, color="orangered"),
+                    name="tokens",
+                    text=df["model_name"],
+                    marker=dict(size=5, color="orange"),
                     showlegend=True,
-                    visible="legendonly",  # Make 'selected words' trace initially visible only in legend
+                    visible="legendonly",  # Make 'words' trace initially visible only in legend
                 )
             )
 
-        # Add trendlines
-        def add_trendline(fig, x, y, name, color, visibility="legendonly"):
-            X = sm.add_constant(df[x])
-            model = sm.OLS(df[y], X).fit()
-            trendline = model.predict(X)
-            fig.add_trace(
-                go.Scatter(
-                    x=df[x],
-                    y=trendline,
-                    mode="lines",
-                    name=f"{name} trendline",
-                    line=dict(color=color, width=2),
-                    visible=visibility,  # Control the initial visibility
+            # Highlight selected models
+            if selected_models:
+                selected_data = df[df["model_name"].isin(selected_models)]
+                fig.add_trace(
+                    go.Scatter(
+                        x=selected_data["num_words_mean"],
+                        y=selected_data[y_column],
+                        mode="markers",
+                        name="selected words",
+                        text=selected_data["model_name"],
+                        marker=dict(size=10, color="blue"),
+                        showlegend=True,
+                    )
                 )
+                fig.add_trace(
+                    go.Scatter(
+                        x=selected_data["num_tokens_mean"],
+                        y=selected_data[y_column],
+                        mode="markers",
+                        name="selected tokens",
+                        text=selected_data["model_name"],
+                        marker=dict(size=10, color="orangered"),
+                        showlegend=True,
+                        visible="legendonly",  # Make 'selected words' trace initially visible only in legend
+                    )
+                )
+
+            # Add trendlines
+            def add_trendline(fig, x, y, name, color, visibility="legendonly"):
+                X = sm.add_constant(df[x])
+                model = sm.OLS(df[y], X).fit()
+                trendline = model.predict(X)
+                fig.add_trace(
+                    go.Scatter(
+                        x=df[x],
+                        y=trendline,
+                        mode="lines",
+                        name=f"{name} trendline",
+                        line=dict(color=color, width=2),
+                        visible=visibility,  # Control the initial visibility
+                    )
+                )
+                return model.rsquared
+
+            r_squared_words = add_trendline(
+                fig, "num_words_mean", y_column, "words", "blue", visibility=True
+            )
+            r_squared_tokens = add_trendline(
+                fig, "num_tokens_mean", y_column, "tokens", "orangered"
             )
-            return model.rsquared
 
-        r_squared_words = add_trendline(
-            fig, "num_words_mean", y_column, "words", "blue", visibility=True
+            # Update layout with titles and labels
+            fig.update_layout(
+                xaxis_title="Mean length",
+                yaxis_title=(
+                    "Win rate"
+                    if y_column == "win_rate"
+                    else (
+                        "LC Win Rate"
+                        if y_column == "length_controlled_winrate"
+                        else "Discrete Win Rate"
+                    )
+                ),
+                title=title,
+                legend_title="Legend",
+            )
+
+            return fig, r_squared_words, r_squared_tokens
+
+        st.markdown("## Overall win rate")
+        y_column1 = "length_controlled_winrate"
+        y_column2 = "win_rate"
+        y_column3 = "discrete_win_rate"
+
+        fig1, r_squared_words_1, r_squared_tokens_1 = create_scatter_plot(
+            df, y_column1, selected_models, "Length-Controlled Win Rate"
         )
-        r_squared_tokens = add_trendline(
-            fig, "num_tokens_mean", y_column, "tokens", "orangered"
+        fig2, r_squared_words_2, r_squared_tokens_2 = create_scatter_plot(
+            df, y_column2, selected_models, "Win Rate"
         )
+        fig3, r_squared_words_3, r_squared_tokens_3 = create_scatter_plot(
+            df, y_column3, selected_models, "Discrete Win Rate"
+        )
+
+        # Create tabs for each chart
+        tab1, tab2, tab3 = st.tabs(["LC Win Rate", "Win Rate", "Discrete Win Rate"])
 
-        # Update layout with titles and labels
-        fig.update_layout(
-            xaxis_title="Mean length",
-            yaxis_title=(
-                "Win rate"
-                if y_column == "win_rate"
-                else (
-                    "LC Win Rate"
-                    if y_column == "length_controlled_winrate"
-                    else "Discrete Win Rate"
+        with tab1:
+            col1, col2 = st.columns([3, 2])
+            col1.plotly_chart(fig1)
+            col2.markdown("#### Rankings")
+            prepped_df = prep_rankings_table(df, "length_controlled_winrate")
+            col2.dataframe(
+                prepped_df,
+                hide_index=True,
+            )
+            with st.expander("Trendline R²"):
+                st.markdown(
+                    f"- R² (Words vs {y_column1}): {r_squared_words_1:.2f} \n- R² (Tokens vs {y_column1}): {r_squared_tokens_1:.2f}"
                 )
+
+        with tab2:
+            col1, col2 = st.columns([3, 2])
+            col1.plotly_chart(fig2)
+            col2.markdown("#### Rankings")
+            prepped_df = prep_rankings_table(df, "win_rate")
+            col2.dataframe(
+                prepped_df,
+                hide_index=True,
+            )
+            with st.expander("Trendline R²"):
+                st.markdown(
+                    f"- R² (Words vs {y_column2}): {r_squared_words_2:.2f} \n- R² (Tokens vs {y_column2}): {r_squared_tokens_2:.2f}"
+                )
+
+        with tab3:
+            col1, col2 = st.columns([3, 2])
+            col1.plotly_chart(fig3)
+            col2.markdown("#### Rankings")
+            prepped_df = prep_rankings_table(df, "discrete_win_rate")
+            col2.dataframe(
+                prepped_df,
+                hide_index=True,
+            )
+            with st.expander("Trendline R²"):
+                st.markdown(
+                    f"- R² (Words vs {y_column3}): {r_squared_words_3:.2f}\n- R² (Tokens vs {y_column3}): {r_squared_tokens_3:.2f}"
+                )
+
+        st.markdown("## Length bias in battles")
+
+        df_response_judging_copy = df_response_judging.copy()
+        if not selected_models:
+            df_response_judging_copy["output_1_num_words"] = df_response_judging_copy[
+                "output_1"
+            ].apply(lambda x: len(x.split()))
+            df_response_judging_copy["output_2_num_words"] = df_response_judging_copy[
+                "output_2"
+            ].apply(lambda x: len(x.split()))
+            df_response_judging_copy["output_num_words_diff"] = (
+                df_response_judging_copy["output_1_num_words"]
+                - df_response_judging_copy["output_2_num_words"]
+            )
+            df_response_judging_copy["assigned_preference"] = (
+                df_response_judging_copy["preference"]
+                .round(0)
+                .apply(get_preference_from_rounded_score)
+            )
+        else:
+            df_response_judging_copy = df_response_judging_copy[
+                df_response_judging_copy["generator_2"].isin(selected_models)
+            ]
+            df_response_judging_copy["output_1_num_words"] = df_response_judging_copy[
+                "output_1"
+            ].apply(lambda x: len(x.split()))
+            df_response_judging_copy["output_2_num_words"] = df_response_judging_copy[
+                "output_2"
+            ].apply(lambda x: len(x.split()))
+            df_response_judging_copy["output_num_words_diff"] = (
+                df_response_judging_copy["output_1_num_words"]
+                - df_response_judging_copy["output_2_num_words"]
+            )
+            df_response_judging_copy["assigned_preference"] = (
+                df_response_judging_copy["preference"]
+                .round(0)
+                .apply(get_preference_from_rounded_score)
+            )
+
+        col1, col2 = st.columns(2)
+        fig = px.scatter(
+            df_response_judging_copy,
+            x="output_1_num_words",
+            y="output_2_num_words",
+            color="assigned_preference",
+            title=f"Pairwise preference based on response length",
+            labels={
+                "output_1_num_words": f"{fixed_model} (1) number of words",
+                "output_2_num_words": "Target model (2) number of words",
+            },
+            color_discrete_map={
+                "[1>2]": "blue",
+                "[2>1]": "orangered",
+                "[1=2]": "green",
+            },
+        )
+        col1.plotly_chart(fig)
+
+        # Plot of output_num_words_diff histogram, colored by assigned_preference.
+        fig = px.histogram(
+            df_response_judging_copy,
+            x="output_num_words_diff",
+            color="assigned_preference",
+            title=f"Pairwise preference counts based on difference in response length",
+            color_discrete_map={
+                "[1>2]": "blue",
+                "[2>1]": "orangered",
+                "[1=2]": "green",
+            },
+            range_x=[-500, 500],
+            labels={
+                "output_num_words_diff": "Length difference in words between gpt4_1106_preview and target model"
+            },
+        )
+        col2.plotly_chart(fig)
+
+        with st.expander("Raw data"):
+            st.dataframe(df)
+
+    # Data explorer
+    with outer_tabs[1]:
+        # Add randomize button at the top of the app
+        st.markdown("## Choose example")
+        st.button(
+            ":game_die: Randomize!",
+            on_click=randomize_selection,
+            type="primary",
+        )
+
+        left_col, right_col = st.columns([1, 3])
+
+        st.session_state.selected_dataset = left_col.selectbox(
+            "Select Dataset",
+            ["all"] + df_response_judging["dataset"].dropna().unique().tolist(),
+            key="dataset_selector",
+            on_change=update_instruction_options,
+        )
+        update_instruction_options()
+        st.session_state.selected_instruction = right_col.selectbox(
+            f"Select Instruction ({len(st.session_state.instruction_options)} unique instructions)",
+            st.session_state.instruction_options,
+            key="instruction_selector",
+            on_change=update_instruction,
+            index=(
+                st.session_state.instruction_options.index(
+                    st.session_state.selected_instruction
+                )
+                if st.session_state.selected_instruction
+                in st.session_state.instruction_options
+                else 0
             ),
-            title=title,
-            legend_title="Legend",
         )
 
-        return fig, r_squared_words, r_squared_tokens
+        # All the models.
+        all_models_judgings_details = df_response_judging[
+            (df_response_judging["generator_1"] == fixed_model)
+            & (
+                df_response_judging["instruction"]
+                == st.session_state.selected_instruction
+            )
+        ]
 
-    y_column1 = "length_controlled_winrate"
-    y_column2 = "win_rate"
-    y_column3 = "discrete_win_rate"
+        st.divider()
 
-    fig1, r_squared_words_1, r_squared_tokens_1 = create_scatter_plot(
-        df, y_column1, selected_models, "Length-Controlled Win Rate"
-    )
-    fig2, r_squared_words_2, r_squared_tokens_2 = create_scatter_plot(
-        df, y_column2, selected_models, "Win Rate"
-    )
-    fig3, r_squared_words_3, r_squared_tokens_3 = create_scatter_plot(
-        df, y_column3, selected_models, "Discrete Win Rate"
-    )
+        st.markdown(f"## Selected instruction")
+        st.info(st.session_state.selected_instruction)
+
+        st.divider()
+
+        st.markdown(f"## Overall Battles")
+        all_models_judgings_details["output_1_num_words"] = all_models_judgings_details[
+            "output_1"
+        ].apply(lambda x: len(x.split()))
+        all_models_judgings_details["output_2_num_words"] = all_models_judgings_details[
+            "output_2"
+        ].apply(lambda x: len(x.split()))
+        all_models_judgings_details["output_num_words_diff"] = (
+            all_models_judgings_details["output_1_num_words"]
+            - all_models_judgings_details["output_2_num_words"]
+        )
+        all_models_judgings_details["assigned_preference"] = (
+            all_models_judgings_details["preference"]
+            .round(0)
+            .apply(get_preference_from_rounded_score)
+        )
+
+        # st.write(all_models_judgings_details)
+
+        col1, col2, col3 = st.columns(3)
+
+        fig = px.histogram(
+            all_models_judgings_details,
+            x="output_num_words_diff",
+            color="assigned_preference",
+            title=f"Pairwise preference counts based on difference in response length",
+            color_discrete_map={
+                "[1>2]": "blue",
+                "[2>1]": "orangered",
+                "[1=2]": "green",
+            },
+            range_x=[-500, 500],
+            labels={
+                "output_num_words_diff": "Difference in number of words between response 1 and 2.",
+                "assigned_preference": "Assigned Preference",
+            },
+        )
+        col1.plotly_chart(fig)
+
+        # Plot of assigned preference counts.
+        fig = px.histogram(
+            all_models_judgings_details,
+            x="assigned_preference",
+            title=f"Assigned preferences for {fixed_model} vs. all models",
+        )
+        col2.plotly_chart(fig)
+
+        # Models that are better than the fixed model.
+        num_words_for_fixed_model = len(
+            all_models_judgings_details.iloc[0]["output_1"].split()
+        )
+        better_models = all_models_judgings_details[
+            all_models_judgings_details["assigned_preference"] == "[2>1]"
+        ]
+
+        shorter_models = better_models[
+            better_models["output_2_num_words"] <= num_words_for_fixed_model
+        ]
+        longer_models = better_models[
+            better_models["output_2_num_words"] > num_words_for_fixed_model
+        ]
+        col3.markdown(
+            f"### Models that are better than {fixed_model} ({num_words_for_fixed_model})"
+        )
+        if shorter_models.size != 0:
+            shorter_models_string = ""
+            for _, shorter_model in shorter_models.iterrows():
+                if shorter_model["generator_2"] != fixed_model:
+                    shorter_models_string += f"- {shorter_model['generator_2']} ({shorter_model['output_2_num_words']})\n"
+            col3.markdown("**With shorter or equal length responses:**")
+            col3.markdown(shorter_models_string)
+        else:
+            col3.write("None")
+        if longer_models.size != 0:
+            longer_models_string = ""
+            for _, longer_model in longer_models.iterrows():
+                if longer_model["generator_2"] != fixed_model:
+                    longer_models_string += f"- {longer_model['generator_2']} ({longer_model['output_2_num_words']})\n"
+            col3.markdown("**With longer responses:**")
+            col3.markdown(longer_models_string)
+        else:
+            col3.write("None")
+
+        # Judging details.
+        st.markdown(f"## Individual Battle Details")
+        judging_details = df_response_judging[
+            (df_response_judging["generator_1"] == fixed_model)
+            & (df_response_judging["generator_2"] == st.session_state.selected_model)
+            & (
+                df_response_judging["instruction"]
+                == st.session_state.selected_instruction
+            )
+        ]
+
+        # if not judging_details.empty:
+        if not judging_details["preference"].empty:
+            preference = get_preference(judging_details["preference"])
+            if preference == "[1>2]":
+                st.write(
+                    f"**{fixed_model}** is better than **{st.session_state.selected_model}**"
+                )
+            else:
+                st.write(
+                    f"**{st.session_state.selected_model}** is better than **{fixed_model}**"
+                )
+            st.write(
+                f"- **Score:** {judging_details['preference'].round(2).item()}\n- **Assigned preference:** {preference}"
+            )
+
+            with st.expander("Additional information"):
+                st.write(
+                    judging_details[
+                        [
+                            "instruction",
+                            "time_per_example",
+                            "price_per_example",
+                            "raw_completion",
+                        ]
+                    ]
+                )
+
+        # Create two columns for model selectors
+        st.markdown("## Responses")
+        col1, col2 = st.columns(2)
+
+        with col1:
+            st.selectbox(
+                "Reference model",
+                [fixed_model],
+                key="fixed_model",
+            )
+
+            # Get the response string for the fixed model
+            if st.session_state.selected_instruction:
+                preference = get_preference(judging_details["preference"])
+                response_details_fixed = df_response_judging[
+                    (
+                        df_response_judging["instruction"]
+                        == st.session_state.selected_instruction
+                    )
+                    & (df_response_judging["generator_1"] == fixed_model)
+                ].iloc[0]
+
+                st.write(
+                    f'Number of words: {len(response_details_fixed["output_1"].split())}'
+                )
+
+                # Display the response string
+                if preference == "[1>2]":
+                    st.success(response_details_fixed["output_1"])
+                else:
+                    st.error(response_details_fixed["output_1"])
+
+        with col2:
+            st.session_state.selected_model = st.selectbox(
+                "Select Model",
+                model_options,
+                key="model_selector",
+                on_change=update_model,
+                index=(
+                    model_options.index(st.session_state.selected_model)
+                    if st.session_state.selected_model
+                    else 0
+                ),
+            )
+
+            # Get the response string for the selected model
+            if (
+                st.session_state.selected_model
+                and st.session_state.selected_instruction
+            ):
+                response_details_dynamic = df_response_judging[
+                    (
+                        df_response_judging["instruction"]
+                        == st.session_state.selected_instruction
+                    )
+                    & (
+                        df_response_judging["generator_2"]
+                        == st.session_state.selected_model
+                    )
+                ].iloc[0]
+
+                st.write(
+                    f'Number of words: {len(response_details_dynamic["output_2"].split())}'
+                )
 
-    # Create tabs for each chart
-    tab1, tab2, tab3 = st.tabs(["LC Win Rate", "Win Rate", "Discrete Win Rate"])
-
-    with tab1:
-        col1, col2 = st.columns([3, 2])
-        col1.plotly_chart(fig1)
-        col2.markdown("#### Rankings")
-        prepped_df = prep_rankings_table(df, "length_controlled_winrate")
-        col2.dataframe(
-            prepped_df,
-            hide_index=True,
-        )
-        with st.expander("Trendline R²"):
-            st.markdown(
-                f"- R² (Words vs {y_column1}): {r_squared_words_1:.2f} \n- R² (Tokens vs {y_column1}): {r_squared_tokens_1:.2f}"
-            )
-
-    with tab2:
-        col1, col2 = st.columns([3, 2])
-        col1.plotly_chart(fig2)
-        col2.markdown("#### Rankings")
-        prepped_df = prep_rankings_table(df, "win_rate")
-        col2.dataframe(
-            prepped_df,
-            hide_index=True,
-        )
-        with st.expander("Trendline R²"):
-            st.markdown(
-                f"- R² (Words vs {y_column2}): {r_squared_words_2:.2f} \n- R² (Tokens vs {y_column2}): {r_squared_tokens_2:.2f}"
-            )
-
-    with tab3:
-        col1, col2 = st.columns([3, 2])
-        col1.plotly_chart(fig3)
-        col2.markdown("#### Rankings")
-        prepped_df = prep_rankings_table(df, "discrete_win_rate")
-        col2.dataframe(
-            prepped_df,
-            hide_index=True,
-        )
-        with st.expander("Trendline R²"):
-            st.markdown(
-                f"- R² (Words vs {y_column3}): {r_squared_words_3:.2f}\n- R² (Tokens vs {y_column3}): {r_squared_tokens_3:.2f}"
-            )
-
-    with st.expander("Raw data"):
-        st.dataframe(df)
+                # Display the response string
+                if preference == "[2>1]":
+                    st.success(response_details_dynamic["output_2"])
+                else:
+                    st.error(response_details_dynamic["output_2"])
 
 
 if __name__ == "__main__":

data/df_response_judging.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5514fedae0d0375f5711fad6fdbf37c8e5e09178d16c29052db0256d09cf2240
+size 214865144

data/df_responses.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbdf08ff250eb5104ee69244672a4baa184b8ae2f928b5c56344102d20926c0d
+size 89555570

data/model_win_rates.json

@@ -1 +0,0 @@
-{"num_words_mean":{"gpt-3.5-turbo-0301":136,"gpt-3.5-turbo-1106_verbose":166,"vicuna-13b-v1.5-togetherai":177,"Qwen1.5-1.8B-Chat":426,"recycled-wizardlm-7b-v1.0":235,"aligner-2b_claude-3-opus-20240229":257,"Qwen1.5-110B-Chat":253,"claude-3-opus-20240229":216,"llama-2-7b-chat-hf":241,"mistral-medium":241,"vicuna-33b-v1.3":237,"cohere":315,"claude-2":174,"guanaco-65b":203,"Mixtral-8x7B-Instruct-v0.1":238,"openchat-v2-w-13b":249,"falcon-7b-instruct":74,"wizardlm-13b-v1.1":244,"Meta-Llama-3-8B-Instruct":301,"FsfairX-Zephyr-Chat-v0.1":342,"Infinity-Instruct-3M-0613-Mistral-7B":79,"Qwen1.5-72B-Chat":243,"xwinlm-7b-v0.1":304,"Mixtral-8x22B-Instruct-v0.1":229,"vicuna-13b-v1.5":177,"dbrx-instruct":235,"zephyr-7b-alpha":208,"tulu-2-dpo-13b":257,"Qwen1.5-7B-Chat":254,"Together-MoA-Lite":297,"cut-13b":269,"Meta-Llama-3-70B-Instruct":299,"vicuna-13b-v1.3":189,"claude-instant-1.2":179,"airoboros-65b":236,"openbuddy-llama2-13b-v11.1":177,"phi-2":102,"Together-MoA":272,"mistral-large-2402":218,"openbuddy-llama-30b-v7.1":162,"TempNet-LLaMA2-Chat-70B-v0.1":296,"pairrm-tulu-2-13b":236,"recycled-wizardlm-7b-v2.0":251,"Storm-7B-best-of-64":336,"vicuna-7b":175,"claude-3-sonnet-20240229":221,"Mistral-7B-Instruct-v0.2":261,"Samba-CoE-v0.1":190,"claude":176,"Nanbeige2-8B-Chat":415,"REBEL-Llama-3-8B-Instruct":341,"chatglm2-6b":175,"gpt-4o-2024-05-13":286,"gpt4_1106_preview_verbose":378,"TempNet-LLaMA2-Chat-13B-v0.1":253,"text_davinci_001":50,"Mixtral-8x7B-Instruct-v0.1_verbose":336,"baize-v2-7b":189,"phi-2-dpo":270,"alpaca-farm-ppo-human":135,"Nanbeige2-16B-Chat":284,"gpt4_0613":183,"pythia-12b-mix-sft":147,"alpaca-7b-neft":170,"Qwen1.5-14B-Chat":252,"gpt-4-0125-preview":313,"guanaco-33b":220,"oasst-sft-llama-33b":125,"gpt4_0613_verbose":237,"llama-2-chat-7b-evol70k-neft":260,"gpt35_turbo_instruct":166,"platolm-7b":209,"llama-2-13b-chat-hf":249,"Nanbeige-Plus-Chat-v0.1":391,"openchat-v2-13b":249,"mistral-orpo-beta":263,"Snorkel-Mistral-PairRM-DPO-best-of-16":393,"tulu-2-dpo-7b":278,"alpaca-7b_verbose":90,"OpenHermes-2.5-Mistral-7B":176,"claude-2.1_verbose":228,"ultralm-13b-v2.0":231,"deita-7b-v1.0":228,"minichat-1.5-3b":242,"Qwen-14B-Chat":167,"airoboros-33b":238,"alpaca-farm-ppo-sim-gpt4-20k":82,"ultralm-13b":181,"openbuddy-falcon-40b-v9":182,"openchat8192-13b":268,"wizardlm-13b":163,"vicuna-13b":175,"merlinite-7B-AOT":281,"gpt4_0314":215,"gpt4_0613_concise":99,"jina-chat":106,"Contextual-KTO-Mistral-PairRM":381,"xwinlm-13b-v0.1":300,"LMCocktail-10.7B-v1":182,"SPPO-Mistral7B-PairRM-ExPO":344,"Mixtral-8x7B-Instruct-v0.1_concise":144,"gpt4_1106_preview_concise":177,"Mistral-7B-ReMax-v0.1":234,"Llama-3-Instruct-8B-SimPO-ExPO":261,"dolphin-2.2.1-mistral-7b":182,"humpback-llama2-70b":178,"openpipe-moa-gpt-4-turbo-v1":272,"vicuna-7b-v1.5":181,"Starling-LM-7B-alpha":301,"falcon-40b-instruct":109,"Samba-CoE-v0.2-best-of-16":225,"opencoderplus-15b":262,"xwinlm-70b-v0.1":282,"wizardlm-13b-v1.2":264,"aligner-2b_qwen1.5-72b-chat":280,"internlm2-chat-7b-ExPO":364,"claude-2.1":177,"vicuna-7b-v1.3":184,"oasst-rlhf-llama-33b":181,"zephyr-7b-alpha-ExPO":201,"openchat-v3.1-13b":235,"SPPO-Llama-3-Instruct-8B-PairRM":317,"minotaur-13b":138,"tulu-2-dpo-13b-ExPO":264,"zephyr-7b-beta-ExPO":224,"tulu-2-dpo-7b-ExPO":277,"Llama-3-Instruct-8B-SimPO":272,"baize-v2-13b":155,"guanaco-7b":233,"ultralm-13b-v2.0-best-of-16":273,"claude-2.1_concise":91,"openchat-13b":260,"tulu-2-dpo-70b":231,"deepseek-llm-67b-chat":189,"humpback-llama-65b":196,"tulu-2-dpo-70b-ExPO":276,"TempNet-LLaMA2-Chat-7B-v0.1":246,"nous-hermes-13b":139,"gpt-3.5-turbo-0613":207,"alpaca-7b_concise":58,"baichuan-13b-chat":225,"claude-3-5-sonnet-20240620":228,"gpt-3.5-turbo-1106":126,"minichat-3b":145,"Storm-7B":300,"oasst-sft-pythia-12b":118,"Conifer-7B-DPO":205,"Snorkel-Mistral-PairRM-DPO":414,"internlm2-chat-20b-ExPO":507,"Samba-CoE-v0.2":210,"gemini-pro":228,"pairrm-tulu-2-70b":264,"text_davinci_003":52,"gpt4":215,"Yi-34B-Chat":339,"Starling-LM-7B-beta-ExPO":336,"pairrm-Yi-34B-Chat":349,"gpt4_1106_preview":323,"evo-7b":280,"zephyr-7b-beta":229,"guanaco-13b":308,"alpaca-7b":66,"internlm2-chat-20b-ppo":371,"gemma-2b-it":165,"pairrm-zephyr-7b-beta":236,"evo-v2-7b":274,"causallm-14b":228,"SPPO-Mistral7B-PairRM":322,"gpt-3.5-turbo-1106_concise":68,"openbuddy-llama-65b-v8":194,"claude2-alpaca-13b":186,"Starling-LM-7B-alpha-ExPO":288,"openbuddy-falcon-7b-v6":192,"gemma-7b-it":176,"phi-2-sft":175,"gpt4_gamed":11,"llama-2-70b-chat-hf":292,"openbuddy-llama2-70b-v10.1":178,"wizardlm-70b":249,"ultralm-13b-best-of-16":311},"num_words_std":{"gpt-3.5-turbo-0301":109,"gpt-3.5-turbo-1106_verbose":110,"vicuna-13b-v1.5-togetherai":119,"Qwen1.5-1.8B-Chat":358,"recycled-wizardlm-7b-v1.0":129,"aligner-2b_claude-3-opus-20240229":145,"Qwen1.5-110B-Chat":153,"claude-3-opus-20240229":113,"llama-2-7b-chat-hf":134,"mistral-medium":166,"vicuna-33b-v1.3":149,"cohere":179,"claude-2":90,"guanaco-65b":157,"Mixtral-8x7B-Instruct-v0.1":163,"openchat-v2-w-13b":151,"falcon-7b-instruct":92,"wizardlm-13b-v1.1":156,"Meta-Llama-3-8B-Instruct":167,"FsfairX-Zephyr-Chat-v0.1":187,"Infinity-Instruct-3M-0613-Mistral-7B":78,"Qwen1.5-72B-Chat":144,"xwinlm-7b-v0.1":207,"Mixtral-8x22B-Instruct-v0.1":168,"vicuna-13b-v1.5":122,"dbrx-instruct":143,"zephyr-7b-alpha":189,"tulu-2-dpo-13b":309,"Qwen1.5-7B-Chat":143,"Together-MoA-Lite":161,"cut-13b":159,"Meta-Llama-3-70B-Instruct":171,"vicuna-13b-v1.3":119,"claude-instant-1.2":104,"airoboros-65b":288,"openbuddy-llama2-13b-v11.1":122,"phi-2":209,"Together-MoA":151,"mistral-large-2402":148,"openbuddy-llama-30b-v7.1":115,"TempNet-LLaMA2-Chat-70B-v0.1":196,"pairrm-tulu-2-13b":135,"recycled-wizardlm-7b-v2.0":134,"Storm-7B-best-of-64":195,"vicuna-7b":124,"claude-3-sonnet-20240229":129,"Mistral-7B-Instruct-v0.2":201,"Samba-CoE-v0.1":138,"claude":97,"Nanbeige2-8B-Chat":173,"REBEL-Llama-3-8B-Instruct":273,"chatglm2-6b":225,"gpt-4o-2024-05-13":190,"gpt4_1106_preview_verbose":190,"TempNet-LLaMA2-Chat-13B-v0.1":136,"text_davinci_001":51,"Mixtral-8x7B-Instruct-v0.1_verbose":169,"baize-v2-7b":133,"phi-2-dpo":137,"alpaca-farm-ppo-human":125,"Nanbeige2-16B-Chat":151,"gpt4_0613":124,"pythia-12b-mix-sft":122,"alpaca-7b-neft":82,"Qwen1.5-14B-Chat":145,"gpt-4-0125-preview":180,"guanaco-33b":170,"oasst-sft-llama-33b":115,"gpt4_0613_verbose":127,"llama-2-chat-7b-evol70k-neft":113,"gpt35_turbo_instruct":157,"platolm-7b":145,"llama-2-13b-chat-hf":135,"Nanbeige-Plus-Chat-v0.1":159,"openchat-v2-13b":151,"mistral-orpo-beta":161,"Snorkel-Mistral-PairRM-DPO-best-of-16":185,"tulu-2-dpo-7b":415,"alpaca-7b_verbose":68,"OpenHermes-2.5-Mistral-7B":154,"claude-2.1_verbose":94,"ultralm-13b-v2.0":137,"deita-7b-v1.0":160,"minichat-1.5-3b":157,"Qwen-14B-Chat":124,"airoboros-33b":246,"alpaca-farm-ppo-sim-gpt4-20k":53,"ultralm-13b":108,"openbuddy-falcon-40b-v9":124,"openchat8192-13b":182,"wizardlm-13b":117,"vicuna-13b":115,"merlinite-7B-AOT":128,"gpt4_0314":160,"gpt4_0613_concise":79,"jina-chat":76,"Contextual-KTO-Mistral-PairRM":205,"xwinlm-13b-v0.1":188,"LMCocktail-10.7B-v1":118,"SPPO-Mistral7B-PairRM-ExPO":144,"Mixtral-8x7B-Instruct-v0.1_concise":131,"gpt4_1106_preview_concise":130,"Mistral-7B-ReMax-v0.1":132,"Llama-3-Instruct-8B-SimPO-ExPO":134,"dolphin-2.2.1-mistral-7b":167,"humpback-llama2-70b":133,"openpipe-moa-gpt-4-turbo-v1":137,"vicuna-7b-v1.5":117,"Starling-LM-7B-alpha":194,"falcon-40b-instruct":111,"Samba-CoE-v0.2-best-of-16":140,"opencoderplus-15b":194,"xwinlm-70b-v0.1":169,"wizardlm-13b-v1.2":188,"aligner-2b_qwen1.5-72b-chat":151,"internlm2-chat-7b-ExPO":172,"claude-2.1":92,"vicuna-7b-v1.3":117,"oasst-rlhf-llama-33b":174,"zephyr-7b-alpha-ExPO":149,"openchat-v3.1-13b":156,"SPPO-Llama-3-Instruct-8B-PairRM":148,"minotaur-13b":109,"tulu-2-dpo-13b-ExPO":139,"zephyr-7b-beta-ExPO":157,"tulu-2-dpo-7b-ExPO":144,"Llama-3-Instruct-8B-SimPO":141,"baize-v2-13b":110,"guanaco-7b":200,"ultralm-13b-v2.0-best-of-16":131,"claude-2.1_concise":68,"openchat-13b":168,"tulu-2-dpo-70b":151,"deepseek-llm-67b-chat":122,"humpback-llama-65b":122,"tulu-2-dpo-70b-ExPO":140,"TempNet-LLaMA2-Chat-7B-v0.1":140,"nous-hermes-13b":130,"gpt-3.5-turbo-0613":149,"alpaca-7b_concise":45,"baichuan-13b-chat":301,"claude-3-5-sonnet-20240620":142,"gpt-3.5-turbo-1106":103,"minichat-3b":119,"Storm-7B":135,"oasst-sft-pythia-12b":120,"Conifer-7B-DPO":126,"Snorkel-Mistral-PairRM-DPO":241,"internlm2-chat-20b-ExPO":206,"Samba-CoE-v0.2":138,"gemini-pro":164,"pairrm-tulu-2-70b":212,"text_davinci_003":66,"gpt4":156,"Yi-34B-Chat":178,"Starling-LM-7B-beta-ExPO":106,"pairrm-Yi-34B-Chat":189,"gpt4_1106_preview":181,"evo-7b":176,"zephyr-7b-beta":160,"guanaco-13b":288,"alpaca-7b":52,"internlm2-chat-20b-ppo":268,"gemma-2b-it":128,"pairrm-zephyr-7b-beta":178,"evo-v2-7b":170,"causallm-14b":166,"SPPO-Mistral7B-PairRM":148,"gpt-3.5-turbo-1106_concise":57,"openbuddy-llama-65b-v8":113,"claude2-alpaca-13b":127,"Starling-LM-7B-alpha-ExPO":174,"openbuddy-falcon-7b-v6":126,"gemma-7b-it":118,"phi-2-sft":123,"gpt4_gamed":60,"llama-2-70b-chat-hf":174,"openbuddy-llama2-70b-v10.1":118,"wizardlm-70b":144,"ultralm-13b-best-of-16":132},"win_rate":{"gpt-3.5-turbo-0301":9.6224532951,"gpt-3.5-turbo-1106_verbose":12.7631698103,"vicuna-13b-v1.5-togetherai":6.9582753694,"Qwen1.5-1.8B-Chat":3.7055568157,"recycled-wizardlm-7b-v1.0":6.6327499605,"aligner-2b_claude-3-opus-20240229":34.4633736232,"Qwen1.5-110B-Chat":33.7770952757,"claude-3-opus-20240229":29.1052695333,"llama-2-7b-chat-hf":4.9613395472,"mistral-medium":21.8557725437,"vicuna-33b-v1.3":12.7059479215,"cohere":12.9014552097,"claude-2":17.1882403567,"guanaco-65b":6.8584945134,"Mixtral-8x7B-Instruct-v0.1":18.2553176264,"openchat-v2-w-13b":9.6153441584,"falcon-7b-instruct":2.1466175532,"wizardlm-13b-v1.1":11.2339095729,"Meta-Llama-3-8B-Instruct":22.5699026093,"FsfairX-Zephyr-Chat-v0.1":35.9464864409,"Infinity-Instruct-3M-0613-Mistral-7B":15.7478281307,"Qwen1.5-72B-Chat":26.4982833956,"xwinlm-7b-v0.1":11.2456517378,"Mixtral-8x22B-Instruct-v0.1":22.2101705475,"vicuna-13b-v1.5":6.7221220149,"dbrx-instruct":19.7553327319,"zephyr-7b-alpha":8.3526639682,"tulu-2-dpo-13b":10.1197883883,"Qwen1.5-7B-Chat":11.7709270696,"Together-MoA-Lite":56.5930456223,"cut-13b":10.7790892025,"Meta-Llama-3-70B-Instruct":33.1778569588,"vicuna-13b-v1.3":7.1372403865,"claude-instant-1.2":16.1273996216,"airoboros-65b":9.3889501497,"openbuddy-llama2-13b-v11.1":6.1747164895,"phi-2":2.350209543,"Together-MoA":59.8688062333,"mistral-large-2402":21.4387759814,"openbuddy-llama-30b-v7.1":6.130014614,"TempNet-LLaMA2-Chat-70B-v0.1":15.0518944202,"pairrm-tulu-2-13b":13.8319010168,"recycled-wizardlm-7b-v2.0":7.3371293705,"Storm-7B-best-of-64":63.0409907519,"vicuna-7b":4.1626111623,"claude-3-sonnet-20240229":25.5563252923,"Mistral-7B-Instruct-v0.2":14.7227726577,"Samba-CoE-v0.1":16.8355018701,"claude":16.9853436124,"Nanbeige2-8B-Chat":39.354502072,"REBEL-Llama-3-8B-Instruct":34.3064238313,"chatglm2-6b":2.7621847965,"gpt-4o-2024-05-13":51.3275757825,"gpt4_1106_preview_verbose":64.303601471,"TempNet-LLaMA2-Chat-13B-v0.1":7.7284050659,"text_davinci_001":2.7640052311,"Mixtral-8x7B-Instruct-v0.1_verbose":24.6140630502,"baize-v2-7b":3.4048149775,"phi-2-dpo":7.7570957018,"alpaca-farm-ppo-human":4.100426815,"Nanbeige2-16B-Chat":37.0360860499,"gpt4_0613":15.7550380876,"pythia-12b-mix-sft":2.578090281,"alpaca-7b-neft":3.1321786695,"Qwen1.5-14B-Chat":18.6458143619,"gpt-4-0125-preview":54.9665397329,"guanaco-33b":5.002493725,"oasst-sft-llama-33b":4.7703909916,"gpt4_0613_verbose":23.2373600435,"llama-2-chat-7b-evol70k-neft":7.6023835122,"gpt35_turbo_instruct":8.4624465044,"platolm-7b":6.3208280585,"llama-2-13b-chat-hf":7.7023099579,"Nanbeige-Plus-Chat-v0.1":56.7030097302,"openchat-v2-13b":8.4350756447,"mistral-orpo-beta":12.5654087946,"Snorkel-Mistral-PairRM-DPO-best-of-16":34.8601328913,"tulu-2-dpo-7b":8.1975153845,"alpaca-7b_verbose":2.9331016025,"OpenHermes-2.5-Mistral-7B":10.3404157058,"claude-2.1_verbose":24.3540710901,"ultralm-13b-v2.0":7.5046229557,"deita-7b-v1.0":12.6466394724,"minichat-1.5-3b":6.5534430528,"Qwen-14B-Chat":7.5023334847,"airoboros-33b":9.0531603961,"alpaca-farm-ppo-sim-gpt4-20k":3.4503419871,"ultralm-13b":5.0745903805,"openbuddy-falcon-40b-v9":5.9557428463,"openchat8192-13b":7.472766808,"wizardlm-13b":5.8781525894,"vicuna-13b":5.8311031845,"merlinite-7B-AOT":29.8963508407,"gpt4_0314":22.0732589287,"gpt4_0613_concise":9.4003205746,"jina-chat":7.7861303934,"Contextual-KTO-Mistral-PairRM":33.2273552,"xwinlm-13b-v0.1":17.4279347502,"LMCocktail-10.7B-v1":13.1534309174,"SPPO-Mistral7B-PairRM-ExPO":35.4431306717,"Mixtral-8x7B-Instruct-v0.1_concise":13.7440401548,"gpt4_1106_preview_concise":22.9201944405,"Mistral-7B-ReMax-v0.1":15.999331369,"Llama-3-Instruct-8B-SimPO-ExPO":40.6328540086,"dolphin-2.2.1-mistral-7b":9.0397997282,"humpback-llama2-70b":10.1217715026,"openpipe-moa-gpt-4-turbo-v1":63.1549345123,"vicuna-7b-v1.5":4.7974939392,"Starling-LM-7B-alpha":14.2459235216,"falcon-40b-instruct":3.3429188225,"Samba-CoE-v0.2-best-of-16":26.9882543183,"opencoderplus-15b":7.406222451,"xwinlm-70b-v0.1":21.8129570739,"wizardlm-13b-v1.2":12.0274803428,"aligner-2b_qwen1.5-72b-chat":31.773037737,"internlm2-chat-7b-ExPO":28.0678174371,"claude-2.1":15.7335067364,"vicuna-7b-v1.3":4.6425118575,"oasst-rlhf-llama-33b":6.2964347858,"zephyr-7b-alpha-ExPO":10.5593543457,"openchat-v3.1-13b":11.0822304894,"SPPO-Llama-3-Instruct-8B-PairRM":39.6728609061,"minotaur-13b":5.7389636691,"tulu-2-dpo-13b-ExPO":15.5514054294,"zephyr-7b-beta-ExPO":11.0611168323,"tulu-2-dpo-7b-ExPO":11.529221039,"Llama-3-Instruct-8B-SimPO":40.5297749846,"baize-v2-13b":4.5905453306,"guanaco-7b":2.8800022662,"ultralm-13b-v2.0-best-of-16":13.8533734712,"claude-2.1_concise":9.2271252406,"openchat-13b":8.0223860109,"tulu-2-dpo-70b":15.9828543741,"deepseek-llm-67b-chat":12.0934222649,"humpback-llama-65b":9.4251390478,"tulu-2-dpo-70b-ExPO":22.9806197059,"TempNet-LLaMA2-Chat-7B-v0.1":5.4301432647,"nous-hermes-13b":5.4118789332,"gpt-3.5-turbo-0613":14.0957985739,"alpaca-7b_concise":1.9911763835,"baichuan-13b-chat":1.9921455615,"claude-3-5-sonnet-20240620":40.5602140968,"gpt-3.5-turbo-1106":9.177964562,"minichat-3b":3.0071507064,"Storm-7B":50.2688690553,"oasst-sft-pythia-12b":1.7901140832,"Conifer-7B-DPO":11.3135856492,"Snorkel-Mistral-PairRM-DPO":30.2200527007,"internlm2-chat-20b-ExPO":46.1853674689,"Samba-CoE-v0.2":21.8473786693,"gemini-pro":18.1776445406,"pairrm-tulu-2-70b":18.6389629674,"text_davinci_003":1.9621476654,"gpt4":23.5767893148,"Yi-34B-Chat":29.6599467188,"Starling-LM-7B-beta-ExPO":29.6008518479,"pairrm-Yi-34B-Chat":31.2412829468,"gpt4_1106_preview":50.0,"evo-7b":15.5774373995,"zephyr-7b-beta":10.9928857554,"guanaco-13b":3.4695968597,"alpaca-7b":2.5914505402,"internlm2-chat-20b-ppo":21.7491545005,"gemma-2b-it":3.4019714381,"pairrm-zephyr-7b-beta":12.8412782556,"evo-v2-7b":20.8341130226,"causallm-14b":11.14616087,"SPPO-Mistral7B-PairRM":32.2453123638,"gpt-3.5-turbo-1106_concise":7.4158649776,"openbuddy-llama-65b-v8":8.7706501509,"claude2-alpaca-13b":7.4373513248,"Starling-LM-7B-alpha-ExPO":18.1797559203,"openbuddy-falcon-7b-v6":3.521174372,"gemma-7b-it":6.9372943797,"phi-2-sft":3.9775677752,"gpt4_gamed":3.7383373714,"llama-2-70b-chat-hf":13.8882583437,"openbuddy-llama2-70b-v10.1":8.0964220963,"wizardlm-70b":14.3838960868,"ultralm-13b-best-of-16":11.3073149478},"standard_error":{"gpt-3.5-turbo-0301":0.9129656687,"gpt-3.5-turbo-1106_verbose":1.0442468192,"vicuna-13b-v1.5-togetherai":0.7825381738,"Qwen1.5-1.8B-Chat":0.5811750995,"recycled-wizardlm-7b-v1.0":0.7713329914,"aligner-2b_claude-3-opus-20240229":1.3146665263,"Qwen1.5-110B-Chat":1.3776163154,"claude-3-opus-20240229":1.3941539442,"llama-2-7b-chat-hf":0.6691754517,"mistral-medium":1.2682402187,"vicuna-33b-v1.3":0.9992557843,"cohere":1.0141034031,"claude-2":1.1748282562,"guanaco-65b":0.8048449272,"Mixtral-8x7B-Instruct-v0.1":1.1885585969,"openchat-v2-w-13b":0.8908241711,"falcon-7b-instruct":0.4542257929,"wizardlm-13b-v1.1":0.9502711246,"Meta-Llama-3-8B-Instruct":1.2575802331,"FsfairX-Zephyr-Chat-v0.1":1.4410058098,"Infinity-Instruct-3M-0613-Mistral-7B":1.1194852006,"Qwen1.5-72B-Chat":1.3042361649,"xwinlm-7b-v0.1":0.9455447881,"Mixtral-8x22B-Instruct-v0.1":1.2780740057,"vicuna-13b-v1.5":0.7674173991,"dbrx-instruct":1.2063251121,"zephyr-7b-alpha":0.8664491645,"tulu-2-dpo-13b":0.929813366,"Qwen1.5-7B-Chat":0.9544463489,"Together-MoA-Lite":1.4464848562,"cut-13b":0.9428953579,"Meta-Llama-3-70B-Instruct":1.3886514096,"vicuna-13b-v1.3":0.7846846272,"claude-instant-1.2":1.1341036838,"airoboros-65b":0.8816208133,"openbuddy-llama2-13b-v11.1":0.753544387,"phi-2":0.4496590406,"Together-MoA":1.4343056045,"mistral-large-2402":1.2485232545,"openbuddy-llama-30b-v7.1":0.7645283386,"TempNet-LLaMA2-Chat-70B-v0.1":1.0801507581,"pairrm-tulu-2-13b":1.0835284665,"recycled-wizardlm-7b-v2.0":0.8012012288,"Storm-7B-best-of-64":1.4253258915,"vicuna-7b":0.6135107768,"claude-3-sonnet-20240229":1.3419811052,"Mistral-7B-Instruct-v0.2":1.0785266447,"Samba-CoE-v0.1":1.1180386125,"claude":1.1687959793,"Nanbeige2-8B-Chat":1.4524224246,"REBEL-Llama-3-8B-Instruct":1.3914900256,"chatglm2-6b":0.5020758951,"gpt-4o-2024-05-13":1.470009459,"gpt4_1106_preview_verbose":1.3348590089,"TempNet-LLaMA2-Chat-13B-v0.1":0.8268032188,"text_davinci_001":0.5177668864,"Mixtral-8x7B-Instruct-v0.1_verbose":1.2975757386,"baize-v2-7b":0.5826293992,"phi-2-dpo":0.8357079426,"alpaca-farm-ppo-human":0.6304721407,"Nanbeige2-16B-Chat":1.4340261273,"gpt4_0613":1.0754642482,"pythia-12b-mix-sft":0.5127326717,"alpaca-7b-neft":0.5522241753,"Qwen1.5-14B-Chat":1.1351340211,"gpt-4-0125-preview":1.4286740089,"guanaco-33b":0.6697115752,"oasst-sft-llama-33b":0.6385940189,"gpt4_0613_verbose":1.2835395056,"llama-2-chat-7b-evol70k-neft":0.8110538776,"gpt35_turbo_instruct":0.8724086934,"platolm-7b":0.7405704765,"llama-2-13b-chat-hf":0.8286143394,"Nanbeige-Plus-Chat-v0.1":1.482841875,"openchat-v2-13b":0.8235980231,"mistral-orpo-beta":0.9929774686,"Snorkel-Mistral-PairRM-DPO-best-of-16":1.3599450437,"tulu-2-dpo-7b":0.8749615125,"alpaca-7b_verbose":0.5302092824,"OpenHermes-2.5-Mistral-7B":0.9356553899,"claude-2.1_verbose":1.29358621,"ultralm-13b-v2.0":0.8150376948,"deita-7b-v1.0":1.0352555321,"minichat-1.5-3b":0.7674159339,"Qwen-14B-Chat":0.8147265702,"airoboros-33b":0.8607792116,"alpaca-farm-ppo-sim-gpt4-20k":0.5834901038,"ultralm-13b":0.6707048924,"openbuddy-falcon-40b-v9":0.7388621614,"openchat8192-13b":0.8038094305,"wizardlm-13b":0.704420227,"vicuna-13b":0.7422829864,"merlinite-7B-AOT":1.3666520485,"gpt4_0314":1.2466725495,"gpt4_0613_concise":0.9010212759,"jina-chat":0.8398450576,"Contextual-KTO-Mistral-PairRM":1.3779687478,"xwinlm-13b-v0.1":1.1450161467,"LMCocktail-10.7B-v1":1.0457195357,"SPPO-Mistral7B-PairRM-ExPO":1.3981308966,"Mixtral-8x7B-Instruct-v0.1_concise":1.0718682992,"gpt4_1106_preview_concise":1.2325177143,"Mistral-7B-ReMax-v0.1":1.1288683901,"Llama-3-Instruct-8B-SimPO-ExPO":1.4439449942,"dolphin-2.2.1-mistral-7b":0.8892901247,"humpback-llama2-70b":0.9401806122,"openpipe-moa-gpt-4-turbo-v1":1.4229800988,"vicuna-7b-v1.5":0.6655960677,"Starling-LM-7B-alpha":1.0685460609,"falcon-40b-instruct":0.5541127159,"Samba-CoE-v0.2-best-of-16":1.318903,"opencoderplus-15b":0.8024858021,"xwinlm-70b-v0.1":1.2303274476,"wizardlm-13b-v1.2":0.9717618177,"aligner-2b_qwen1.5-72b-chat":1.2392772646,"internlm2-chat-7b-ExPO":1.3159792318,"claude-2.1":1.1203158654,"vicuna-7b-v1.3":0.6420919828,"oasst-rlhf-llama-33b":0.7417944201,"zephyr-7b-alpha-ExPO":0.9774634449,"openchat-v3.1-13b":0.9501308701,"SPPO-Llama-3-Instruct-8B-PairRM":1.4247223562,"minotaur-13b":0.7271241247,"tulu-2-dpo-13b-ExPO":1.1714853384,"zephyr-7b-beta-ExPO":1.0204784889,"tulu-2-dpo-7b-ExPO":1.0497814893,"Llama-3-Instruct-8B-SimPO":1.4225744647,"baize-v2-13b":0.6497033227,"guanaco-7b":0.5202924149,"ultralm-13b-v2.0-best-of-16":1.049344706,"claude-2.1_concise":0.8921752289,"openchat-13b":0.8368334957,"tulu-2-dpo-70b":1.1457861368,"deepseek-llm-67b-chat":1.0173843633,"humpback-llama-65b":0.9300866723,"tulu-2-dpo-70b-ExPO":1.3591734083,"TempNet-LLaMA2-Chat-7B-v0.1":0.7210775889,"nous-hermes-13b":0.7081240036,"gpt-3.5-turbo-0613":1.0371186215,"alpaca-7b_concise":0.4437510224,"baichuan-13b-chat":0.4176985079,"claude-3-5-sonnet-20240620":1.4679655404,"gpt-3.5-turbo-1106":0.8904117512,"minichat-3b":0.5041245962,"Storm-7B":1.4728176781,"oasst-sft-pythia-12b":0.3985580883,"Conifer-7B-DPO":0.9870897936,"Snorkel-Mistral-PairRM-DPO":1.3328273013,"internlm2-chat-20b-ExPO":1.4638315246,"Samba-CoE-v0.2":1.2171089783,"gemini-pro":1.1588503791,"pairrm-tulu-2-70b":1.19249667,"text_davinci_003":0.4346747594,"gpt4":1.2757042012,"Yi-34B-Chat":1.3225712598,"Starling-LM-7B-beta-ExPO":1.3252049543,"pairrm-Yi-34B-Chat":1.3482437399,"gpt4_1106_preview":0.0,"evo-7b":1.0835570389,"zephyr-7b-beta":0.9617876718,"guanaco-13b":0.5518606726,"alpaca-7b":0.4870855383,"internlm2-chat-20b-ppo":1.244366241,"gemma-2b-it":0.538998125,"pairrm-zephyr-7b-beta":1.0535874942,"evo-v2-7b":1.2159901798,"causallm-14b":0.9544127301,"SPPO-Mistral7B-PairRM":1.390800011,"gpt-3.5-turbo-1106_concise":0.8374438114,"openbuddy-llama-65b-v8":0.8871992619,"claude2-alpaca-13b":0.8249428868,"Starling-LM-7B-alpha-ExPO":1.2498324796,"openbuddy-falcon-7b-v6":0.5655836443,"gemma-7b-it":0.7869665732,"phi-2-sft":0.6098271417,"gpt4_gamed":0.6278799634,"llama-2-70b-chat-hf":1.0799847727,"openbuddy-llama2-70b-v10.1":0.8498371494,"wizardlm-70b":1.0395048913,"ultralm-13b-best-of-16":0.9418434059},"n_wins":{"gpt-3.5-turbo-0301":71.0,"gpt-3.5-turbo-1106_verbose":94.0,"vicuna-13b-v1.5-togetherai":53.0,"Qwen1.5-1.8B-Chat":27.0,"recycled-wizardlm-7b-v1.0":53.0,"aligner-2b_claude-3-opus-20240229":225.0,"Qwen1.5-110B-Chat":255.0,"claude-3-opus-20240229":223.0,"llama-2-7b-chat-hf":38.0,"mistral-medium":164.0,"vicuna-33b-v1.3":90.0,"cohere":96.0,"claude-2":131.0,"guanaco-65b":54.0,"Mixtral-8x7B-Instruct-v0.1":135.0,"openchat-v2-w-13b":67.0,"falcon-7b-instruct":16.0,"wizardlm-13b-v1.1":79.0,"Meta-Llama-3-8B-Instruct":176.0,"FsfairX-Zephyr-Chat-v0.1":285.0,"Infinity-Instruct-3M-0613-Mistral-7B":118.0,"Qwen1.5-72B-Chat":201.0,"xwinlm-7b-v0.1":77.0,"Mixtral-8x22B-Instruct-v0.1":174.0,"vicuna-13b-v1.5":48.0,"dbrx-instruct":147.0,"zephyr-7b-alpha":59.0,"tulu-2-dpo-13b":75.0,"Qwen1.5-7B-Chat":80.0,"Together-MoA-Lite":456.0,"cut-13b":83.0,"Meta-Llama-3-70B-Instruct":266.0,"vicuna-13b-v1.3":50.0,"claude-instant-1.2":120.0,"airoboros-65b":67.0,"openbuddy-llama2-13b-v11.1":42.0,"phi-2":15.0,"Together-MoA":490.0,"mistral-large-2402":166.0,"openbuddy-llama-30b-v7.1":47.0,"TempNet-LLaMA2-Chat-70B-v0.1":111.0,"pairrm-tulu-2-13b":110.0,"recycled-wizardlm-7b-v2.0":50.0,"Storm-7B-best-of-64":519.0,"vicuna-7b":28.0,"claude-3-sonnet-20240229":193.0,"Mistral-7B-Instruct-v0.2":113.0,"Samba-CoE-v0.1":124.0,"claude":129.0,"Nanbeige2-8B-Chat":323.0,"REBEL-Llama-3-8B-Instruct":268.0,"chatglm2-6b":19.0,"gpt-4o-2024-05-13":429.0,"gpt4_1106_preview_verbose":525.0,"TempNet-LLaMA2-Chat-13B-v0.1":56.0,"text_davinci_001":23.0,"Mixtral-8x7B-Instruct-v0.1_verbose":194.0,"baize-v2-7b":26.0,"phi-2-dpo":57.0,"alpaca-farm-ppo-human":32.0,"Nanbeige2-16B-Chat":288.0,"gpt4_0613":117.0,"pythia-12b-mix-sft":19.0,"alpaca-7b-neft":22.0,"Qwen1.5-14B-Chat":137.0,"gpt-4-0125-preview":446.0,"guanaco-33b":37.0,"oasst-sft-llama-33b":36.0,"gpt4_0613_verbose":171.0,"llama-2-chat-7b-evol70k-neft":57.0,"gpt35_turbo_instruct":66.0,"platolm-7b":42.0,"llama-2-13b-chat-hf":60.0,"Nanbeige-Plus-Chat-v0.1":456.0,"openchat-v2-13b":56.0,"mistral-orpo-beta":95.0,"Snorkel-Mistral-PairRM-DPO-best-of-16":270.0,"tulu-2-dpo-7b":64.0,"alpaca-7b_verbose":22.0,"OpenHermes-2.5-Mistral-7B":75.0,"claude-2.1_verbose":191.0,"ultralm-13b-v2.0":51.0,"deita-7b-v1.0":96.0,"minichat-1.5-3b":48.0,"Qwen-14B-Chat":57.0,"airoboros-33b":64.0,"alpaca-farm-ppo-sim-gpt4-20k":26.0,"ultralm-13b":38.0,"openbuddy-falcon-40b-v9":45.0,"openchat8192-13b":51.0,"wizardlm-13b":42.0,"vicuna-13b":44.0,"merlinite-7B-AOT":234.0,"gpt4_0314":172.0,"gpt4_0613_concise":71.0,"jina-chat":59.0,"Contextual-KTO-Mistral-PairRM":260.0,"xwinlm-13b-v0.1":129.0,"LMCocktail-10.7B-v1":104.0,"SPPO-Mistral7B-PairRM-ExPO":274.0,"Mixtral-8x7B-Instruct-v0.1_concise":105.0,"gpt4_1106_preview_concise":172.0,"Mistral-7B-ReMax-v0.1":120.0,"Llama-3-Instruct-8B-SimPO-ExPO":325.0,"dolphin-2.2.1-mistral-7b":68.0,"humpback-llama2-70b":77.0,"openpipe-moa-gpt-4-turbo-v1":515.0,"vicuna-7b-v1.5":35.0,"Starling-LM-7B-alpha":102.0,"falcon-40b-instruct":27.0,"Samba-CoE-v0.2-best-of-16":201.0,"opencoderplus-15b":52.0,"xwinlm-70b-v0.1":166.0,"wizardlm-13b-v1.2":82.0,"aligner-2b_qwen1.5-72b-chat":180.0,"internlm2-chat-7b-ExPO":209.0,"claude-2.1":115.0,"vicuna-7b-v1.3":31.0,"oasst-rlhf-llama-33b":44.0,"zephyr-7b-alpha-ExPO":79.0,"openchat-v3.1-13b":80.0,"SPPO-Llama-3-Instruct-8B-PairRM":310.0,"minotaur-13b":42.0,"tulu-2-dpo-13b-ExPO":121.0,"zephyr-7b-beta-ExPO":89.0,"tulu-2-dpo-7b-ExPO":91.0,"Llama-3-Instruct-8B-SimPO":319.0,"baize-v2-13b":32.0,"guanaco-7b":21.0,"ultralm-13b-v2.0-best-of-16":98.0,"claude-2.1_concise":72.0,"openchat-13b":58.0,"tulu-2-dpo-70b":119.0,"deepseek-llm-67b-chat":90.0,"humpback-llama-65b":70.0,"tulu-2-dpo-70b-ExPO":184.0,"TempNet-LLaMA2-Chat-7B-v0.1":39.0,"nous-hermes-13b":43.0,"gpt-3.5-turbo-0613":99.0,"alpaca-7b_concise":15.0,"baichuan-13b-chat":14.0,"claude-3-5-sonnet-20240620":312.0,"gpt-3.5-turbo-1106":64.0,"minichat-3b":22.0,"Storm-7B":397.0,"oasst-sft-pythia-12b":13.0,"Conifer-7B-DPO":87.0,"Snorkel-Mistral-PairRM-DPO":231.0,"internlm2-chat-20b-ExPO":375.0,"Samba-CoE-v0.2":159.0,"gemini-pro":135.0,"pairrm-tulu-2-70b":140.0,"text_davinci_003":14.0,"gpt4":179.0,"Yi-34B-Chat":219.0,"Starling-LM-7B-beta-ExPO":225.0,"pairrm-Yi-34B-Chat":239.0,"gpt4_1106_preview":0.0,"evo-7b":112.0,"zephyr-7b-beta":78.0,"guanaco-13b":22.0,"alpaca-7b":17.0,"internlm2-chat-20b-ppo":170.0,"gemma-2b-it":23.0,"pairrm-zephyr-7b-beta":98.0,"evo-v2-7b":158.0,"causallm-14b":81.0,"SPPO-Mistral7B-PairRM":249.0,"gpt-3.5-turbo-1106_concise":57.0,"openbuddy-llama-65b-v8":64.0,"claude2-alpaca-13b":59.0,"Starling-LM-7B-alpha-ExPO":148.0,"openbuddy-falcon-7b-v6":27.0,"gemma-7b-it":50.0,"phi-2-sft":28.0,"gpt4_gamed":32.0,"llama-2-70b-chat-hf":104.0,"openbuddy-llama2-70b-v10.1":57.0,"wizardlm-70b":106.0,"ultralm-13b-best-of-16":80.0},"n_wins_base":{"gpt-3.5-turbo-0301":733.0,"gpt-3.5-turbo-1106_verbose":709.0,"vicuna-13b-v1.5-togetherai":747.0,"Qwen1.5-1.8B-Chat":774.0,"recycled-wizardlm-7b-v1.0":752.0,"aligner-2b_claude-3-opus-20240229":475.0,"Qwen1.5-110B-Chat":545.0,"claude-3-opus-20240229":579.0,"llama-2-7b-chat-hf":766.0,"mistral-medium":639.0,"vicuna-33b-v1.3":711.0,"cohere":709.0,"claude-2":673.0,"guanaco-65b":751.0,"Mixtral-8x7B-Instruct-v0.1":668.0,"openchat-v2-w-13b":736.0,"falcon-7b-instruct":787.0,"wizardlm-13b-v1.1":723.0,"Meta-Llama-3-8B-Instruct":626.0,"FsfairX-Zephyr-Chat-v0.1":517.0,"Infinity-Instruct-3M-0613-Mistral-7B":687.0,"Qwen1.5-72B-Chat":600.0,"xwinlm-7b-v0.1":727.0,"Mixtral-8x22B-Instruct-v0.1":628.0,"vicuna-13b-v1.5":753.0,"dbrx-instruct":657.0,"zephyr-7b-alpha":745.0,"tulu-2-dpo-13b":728.0,"Qwen1.5-7B-Chat":721.0,"Together-MoA-Lite":347.0,"cut-13b":721.0,"Meta-Llama-3-70B-Instruct":537.0,"vicuna-13b-v1.3":751.0,"claude-instant-1.2":682.0,"airoboros-65b":735.0,"openbuddy-llama2-13b-v11.1":761.0,"phi-2":785.0,"Together-MoA":314.0,"mistral-large-2402":638.0,"openbuddy-llama-30b-v7.1":755.0,"TempNet-LLaMA2-Chat-70B-v0.1":691.0,"pairrm-tulu-2-13b":694.0,"recycled-wizardlm-7b-v2.0":755.0,"Storm-7B-best-of-64":286.0,"vicuna-7b":775.0,"claude-3-sonnet-20240229":608.0,"Mistral-7B-Instruct-v0.2":691.0,"Samba-CoE-v0.1":680.0,"claude":676.0,"Nanbeige2-8B-Chat":480.0,"REBEL-Llama-3-8B-Instruct":537.0,"chatglm2-6b":781.0,"gpt-4o-2024-05-13":369.0,"gpt4_1106_preview_verbose":268.0,"TempNet-LLaMA2-Chat-13B-v0.1":749.0,"text_davinci_001":777.0,"Mixtral-8x7B-Instruct-v0.1_verbose":609.0,"baize-v2-7b":779.0,"phi-2-dpo":748.0,"alpaca-farm-ppo-human":770.0,"Nanbeige2-16B-Chat":514.0,"gpt4_0613":684.0,"pythia-12b-mix-sft":786.0,"alpaca-7b-neft":783.0,"Qwen1.5-14B-Chat":664.0,"gpt-4-0125-preview":347.0,"guanaco-33b":768.0,"oasst-sft-llama-33b":764.0,"gpt4_0613_verbose":630.0,"llama-2-chat-7b-evol70k-neft":748.0,"gpt35_turbo_instruct":735.0,"platolm-7b":759.0,"llama-2-13b-chat-hf":744.0,"Nanbeige-Plus-Chat-v0.1":347.0,"openchat-v2-13b":746.0,"mistral-orpo-beta":707.0,"Snorkel-Mistral-PairRM-DPO-best-of-16":533.0,"tulu-2-dpo-7b":740.0,"alpaca-7b_verbose":778.0,"OpenHermes-2.5-Mistral-7B":727.0,"claude-2.1_verbose":613.0,"ultralm-13b-v2.0":754.0,"deita-7b-v1.0":708.0,"minichat-1.5-3b":757.0,"Qwen-14B-Chat":742.0,"airoboros-33b":740.0,"alpaca-farm-ppo-sim-gpt4-20k":776.0,"ultralm-13b":765.0,"openbuddy-falcon-40b-v9":758.0,"openchat8192-13b":754.0,"wizardlm-13b":759.0,"vicuna-13b":759.0,"merlinite-7B-AOT":571.0,"gpt4_0314":627.0,"gpt4_0613_concise":729.0,"jina-chat":743.0,"Contextual-KTO-Mistral-PairRM":544.0,"xwinlm-13b-v0.1":672.0,"LMCocktail-10.7B-v1":700.0,"SPPO-Mistral7B-PairRM-ExPO":531.0,"Mixtral-8x7B-Instruct-v0.1_concise":700.0,"gpt4_1106_preview_concise":622.0,"Mistral-7B-ReMax-v0.1":683.0,"Llama-3-Instruct-8B-SimPO-ExPO":479.0,"dolphin-2.2.1-mistral-7b":734.0,"humpback-llama2-70b":727.0,"openpipe-moa-gpt-4-turbo-v1":283.0,"vicuna-7b-v1.5":767.0,"Starling-LM-7B-alpha":702.0,"falcon-40b-instruct":777.0,"Samba-CoE-v0.2-best-of-16":601.0,"opencoderplus-15b":750.0,"xwinlm-70b-v0.1":635.0,"wizardlm-13b-v1.2":720.0,"aligner-2b_qwen1.5-72b-chat":473.0,"internlm2-chat-7b-ExPO":595.0,"claude-2.1":688.0,"vicuna-7b-v1.3":771.0,"oasst-rlhf-llama-33b":759.0,"zephyr-7b-alpha-ExPO":725.0,"openchat-v3.1-13b":720.0,"SPPO-Llama-3-Instruct-8B-PairRM":494.0,"minotaur-13b":758.0,"tulu-2-dpo-13b-ExPO":679.0,"zephyr-7b-beta-ExPO":716.0,"tulu-2-dpo-7b-ExPO":714.0,"Llama-3-Instruct-8B-SimPO":485.0,"baize-v2-13b":770.0,"guanaco-7b":783.0,"ultralm-13b-v2.0-best-of-16":705.0,"claude-2.1_concise":730.0,"openchat-13b":746.0,"tulu-2-dpo-70b":683.0,"deepseek-llm-67b-chat":713.0,"humpback-llama-65b":734.0,"tulu-2-dpo-70b-ExPO":620.0,"TempNet-LLaMA2-Chat-7B-v0.1":765.0,"nous-hermes-13b":761.0,"gpt-3.5-turbo-0613":700.0,"alpaca-7b_concise":787.0,"baichuan-13b-chat":790.0,"claude-3-5-sonnet-20240620":493.0,"gpt-3.5-turbo-1106":737.0,"minichat-3b":779.0,"Storm-7B":408.0,"oasst-sft-pythia-12b":790.0,"Conifer-7B-DPO":717.0,"Snorkel-Mistral-PairRM-DPO":572.0,"internlm2-chat-20b-ExPO":430.0,"Samba-CoE-v0.2":645.0,"gemini-pro":665.0,"pairrm-tulu-2-70b":665.0,"text_davinci_003":787.0,"gpt4":618.0,"Yi-34B-Chat":582.0,"Starling-LM-7B-beta-ExPO":580.0,"pairrm-Yi-34B-Chat":563.0,"gpt4_1106_preview":0.0,"evo-7b":689.0,"zephyr-7b-beta":725.0,"guanaco-13b":780.0,"alpaca-7b":785.0,"internlm2-chat-20b-ppo":632.0,"gemma-2b-it":782.0,"pairrm-zephyr-7b-beta":706.0,"evo-v2-7b":644.0,"causallm-14b":720.0,"SPPO-Mistral7B-PairRM":556.0,"gpt-3.5-turbo-1106_concise":744.0,"openbuddy-llama-65b-v8":738.0,"claude2-alpaca-13b":746.0,"Starling-LM-7B-alpha-ExPO":657.0,"openbuddy-falcon-7b-v6":778.0,"gemma-7b-it":754.0,"phi-2-sft":777.0,"gpt4_gamed":771.0,"llama-2-70b-chat-hf":700.0,"openbuddy-llama2-70b-v10.1":744.0,"wizardlm-70b":697.0,"ultralm-13b-best-of-16":723.0},"n_draws":{"gpt-3.5-turbo-0301":1.0,"gpt-3.5-turbo-1106_verbose":2.0,"vicuna-13b-v1.5-togetherai":5.0,"Qwen1.5-1.8B-Chat":3.0,"recycled-wizardlm-7b-v1.0":0.0,"aligner-2b_claude-3-opus-20240229":105.0,"Qwen1.5-110B-Chat":5.0,"claude-3-opus-20240229":3.0,"llama-2-7b-chat-hf":1.0,"mistral-medium":2.0,"vicuna-33b-v1.3":4.0,"cohere":0.0,"claude-2":1.0,"guanaco-65b":0.0,"Mixtral-8x7B-Instruct-v0.1":2.0,"openchat-v2-w-13b":2.0,"falcon-7b-instruct":2.0,"wizardlm-13b-v1.1":3.0,"Meta-Llama-3-8B-Instruct":3.0,"FsfairX-Zephyr-Chat-v0.1":3.0,"Infinity-Instruct-3M-0613-Mistral-7B":0.0,"Qwen1.5-72B-Chat":4.0,"xwinlm-7b-v0.1":1.0,"Mixtral-8x22B-Instruct-v0.1":3.0,"vicuna-13b-v1.5":4.0,"dbrx-instruct":1.0,"zephyr-7b-alpha":1.0,"tulu-2-dpo-13b":2.0,"Qwen1.5-7B-Chat":4.0,"Together-MoA-Lite":2.0,"cut-13b":1.0,"Meta-Llama-3-70B-Instruct":2.0,"vicuna-13b-v1.3":4.0,"claude-instant-1.2":3.0,"airoboros-65b":3.0,"openbuddy-llama2-13b-v11.1":2.0,"phi-2":3.0,"Together-MoA":1.0,"mistral-large-2402":1.0,"openbuddy-llama-30b-v7.1":3.0,"TempNet-LLaMA2-Chat-70B-v0.1":2.0,"pairrm-tulu-2-13b":1.0,"recycled-wizardlm-7b-v2.0":0.0,"Storm-7B-best-of-64":0.0,"vicuna-7b":2.0,"claude-3-sonnet-20240229":4.0,"Mistral-7B-Instruct-v0.2":1.0,"Samba-CoE-v0.1":1.0,"claude":0.0,"Nanbeige2-8B-Chat":2.0,"REBEL-Llama-3-8B-Instruct":0.0,"chatglm2-6b":5.0,"gpt-4o-2024-05-13":7.0,"gpt4_1106_preview_verbose":12.0,"TempNet-LLaMA2-Chat-13B-v0.1":0.0,"text_davinci_001":3.0,"Mixtral-8x7B-Instruct-v0.1_verbose":2.0,"baize-v2-7b":0.0,"phi-2-dpo":0.0,"alpaca-farm-ppo-human":3.0,"Nanbeige2-16B-Chat":3.0,"gpt4_0613":4.0,"pythia-12b-mix-sft":0.0,"alpaca-7b-neft":0.0,"Qwen1.5-14B-Chat":4.0,"gpt-4-0125-preview":12.0,"guanaco-33b":0.0,"oasst-sft-llama-33b":5.0,"gpt4_0613_verbose":4.0,"llama-2-chat-7b-evol70k-neft":0.0,"gpt35_turbo_instruct":3.0,"platolm-7b":2.0,"llama-2-13b-chat-hf":1.0,"Nanbeige-Plus-Chat-v0.1":2.0,"openchat-v2-13b":3.0,"mistral-orpo-beta":3.0,"Snorkel-Mistral-PairRM-DPO-best-of-16":2.0,"tulu-2-dpo-7b":1.0,"alpaca-7b_verbose":2.0,"OpenHermes-2.5-Mistral-7B":3.0,"claude-2.1_verbose":1.0,"ultralm-13b-v2.0":0.0,"deita-7b-v1.0":1.0,"minichat-1.5-3b":0.0,"Qwen-14B-Chat":6.0,"airoboros-33b":1.0,"alpaca-farm-ppo-sim-gpt4-20k":3.0,"ultralm-13b":2.0,"openbuddy-falcon-40b-v9":2.0,"openchat8192-13b":0.0,"wizardlm-13b":4.0,"vicuna-13b":2.0,"merlinite-7B-AOT":0.0,"gpt4_0314":6.0,"gpt4_0613_concise":5.0,"jina-chat":3.0,"Contextual-KTO-Mistral-PairRM":1.0,"xwinlm-13b-v0.1":4.0,"LMCocktail-10.7B-v1":1.0,"SPPO-Mistral7B-PairRM-ExPO":0.0,"Mixtral-8x7B-Instruct-v0.1_concise":0.0,"gpt4_1106_preview_concise":11.0,"Mistral-7B-ReMax-v0.1":2.0,"Llama-3-Instruct-8B-SimPO-ExPO":1.0,"dolphin-2.2.1-mistral-7b":3.0,"humpback-llama2-70b":1.0,"openpipe-moa-gpt-4-turbo-v1":7.0,"vicuna-7b-v1.5":3.0,"Starling-LM-7B-alpha":1.0,"falcon-40b-instruct":1.0,"Samba-CoE-v0.2-best-of-16":3.0,"opencoderplus-15b":3.0,"xwinlm-70b-v0.1":4.0,"wizardlm-13b-v1.2":3.0,"aligner-2b_qwen1.5-72b-chat":152.0,"internlm2-chat-7b-ExPO":1.0,"claude-2.1":2.0,"vicuna-7b-v1.3":3.0,"oasst-rlhf-llama-33b":2.0,"zephyr-7b-alpha-ExPO":1.0,"openchat-v3.1-13b":5.0,"SPPO-Llama-3-Instruct-8B-PairRM":1.0,"minotaur-13b":4.0,"tulu-2-dpo-13b-ExPO":5.0,"zephyr-7b-beta-ExPO":0.0,"tulu-2-dpo-7b-ExPO":0.0,"Llama-3-Instruct-8B-SimPO":1.0,"baize-v2-13b":3.0,"guanaco-7b":1.0,"ultralm-13b-v2.0-best-of-16":2.0,"claude-2.1_concise":3.0,"openchat-13b":1.0,"tulu-2-dpo-70b":3.0,"deepseek-llm-67b-chat":2.0,"humpback-llama-65b":1.0,"tulu-2-dpo-70b-ExPO":1.0,"TempNet-LLaMA2-Chat-7B-v0.1":1.0,"nous-hermes-13b":1.0,"gpt-3.5-turbo-0613":6.0,"alpaca-7b_concise":2.0,"baichuan-13b-chat":1.0,"claude-3-5-sonnet-20240620":0.0,"gpt-3.5-turbo-1106":4.0,"minichat-3b":4.0,"Storm-7B":0.0,"oasst-sft-pythia-12b":2.0,"Conifer-7B-DPO":1.0,"Snorkel-Mistral-PairRM-DPO":1.0,"internlm2-chat-20b-ExPO":0.0,"Samba-CoE-v0.2":1.0,"gemini-pro":5.0,"pairrm-tulu-2-70b":0.0,"text_davinci_003":4.0,"gpt4":8.0,"Yi-34B-Chat":4.0,"Starling-LM-7B-beta-ExPO":0.0,"pairrm-Yi-34B-Chat":3.0,"gpt4_1106_preview":805.0,"evo-7b":4.0,"zephyr-7b-beta":2.0,"guanaco-13b":3.0,"alpaca-7b":3.0,"internlm2-chat-20b-ppo":3.0,"gemma-2b-it":0.0,"pairrm-zephyr-7b-beta":1.0,"evo-v2-7b":3.0,"causallm-14b":4.0,"SPPO-Mistral7B-PairRM":0.0,"gpt-3.5-turbo-1106_concise":4.0,"openbuddy-llama-65b-v8":3.0,"claude2-alpaca-13b":0.0,"Starling-LM-7B-alpha-ExPO":0.0,"openbuddy-falcon-7b-v6":0.0,"gemma-7b-it":1.0,"phi-2-sft":0.0,"gpt4_gamed":2.0,"llama-2-70b-chat-hf":0.0,"openbuddy-llama2-70b-v10.1":4.0,"wizardlm-70b":2.0,"ultralm-13b-best-of-16":2.0},"n_total":{"gpt-3.5-turbo-0301":805.0,"gpt-3.5-turbo-1106_verbose":805.0,"vicuna-13b-v1.5-togetherai":805.0,"Qwen1.5-1.8B-Chat":804.0,"recycled-wizardlm-7b-v1.0":805.0,"aligner-2b_claude-3-opus-20240229":805.0,"Qwen1.5-110B-Chat":805.0,"claude-3-opus-20240229":805.0,"llama-2-7b-chat-hf":805.0,"mistral-medium":805.0,"vicuna-33b-v1.3":805.0,"cohere":805.0,"claude-2":805.0,"guanaco-65b":805.0,"Mixtral-8x7B-Instruct-v0.1":805.0,"openchat-v2-w-13b":805.0,"falcon-7b-instruct":805.0,"wizardlm-13b-v1.1":805.0,"Meta-Llama-3-8B-Instruct":805.0,"FsfairX-Zephyr-Chat-v0.1":805.0,"Infinity-Instruct-3M-0613-Mistral-7B":805.0,"Qwen1.5-72B-Chat":805.0,"xwinlm-7b-v0.1":805.0,"Mixtral-8x22B-Instruct-v0.1":805.0,"vicuna-13b-v1.5":805.0,"dbrx-instruct":805.0,"zephyr-7b-alpha":805.0,"tulu-2-dpo-13b":805.0,"Qwen1.5-7B-Chat":805.0,"Together-MoA-Lite":805.0,"cut-13b":805.0,"Meta-Llama-3-70B-Instruct":805.0,"vicuna-13b-v1.3":805.0,"claude-instant-1.2":805.0,"airoboros-65b":805.0,"openbuddy-llama2-13b-v11.1":805.0,"phi-2":803.0,"Together-MoA":805.0,"mistral-large-2402":805.0,"openbuddy-llama-30b-v7.1":805.0,"TempNet-LLaMA2-Chat-70B-v0.1":804.0,"pairrm-tulu-2-13b":805.0,"recycled-wizardlm-7b-v2.0":805.0,"Storm-7B-best-of-64":805.0,"vicuna-7b":805.0,"claude-3-sonnet-20240229":805.0,"Mistral-7B-Instruct-v0.2":805.0,"Samba-CoE-v0.1":805.0,"claude":805.0,"Nanbeige2-8B-Chat":805.0,"REBEL-Llama-3-8B-Instruct":805.0,"chatglm2-6b":805.0,"gpt-4o-2024-05-13":805.0,"gpt4_1106_preview_verbose":805.0,"TempNet-LLaMA2-Chat-13B-v0.1":805.0,"text_davinci_001":803.0,"Mixtral-8x7B-Instruct-v0.1_verbose":805.0,"baize-v2-7b":805.0,"phi-2-dpo":805.0,"alpaca-farm-ppo-human":805.0,"Nanbeige2-16B-Chat":805.0,"gpt4_0613":805.0,"pythia-12b-mix-sft":805.0,"alpaca-7b-neft":805.0,"Qwen1.5-14B-Chat":805.0,"gpt-4-0125-preview":805.0,"guanaco-33b":805.0,"oasst-sft-llama-33b":805.0,"gpt4_0613_verbose":805.0,"llama-2-chat-7b-evol70k-neft":805.0,"gpt35_turbo_instruct":804.0,"platolm-7b":803.0,"llama-2-13b-chat-hf":805.0,"Nanbeige-Plus-Chat-v0.1":805.0,"openchat-v2-13b":805.0,"mistral-orpo-beta":805.0,"Snorkel-Mistral-PairRM-DPO-best-of-16":805.0,"tulu-2-dpo-7b":805.0,"alpaca-7b_verbose":802.0,"OpenHermes-2.5-Mistral-7B":805.0,"claude-2.1_verbose":805.0,"ultralm-13b-v2.0":805.0,"deita-7b-v1.0":805.0,"minichat-1.5-3b":805.0,"Qwen-14B-Chat":805.0,"airoboros-33b":805.0,"alpaca-farm-ppo-sim-gpt4-20k":805.0,"ultralm-13b":805.0,"openbuddy-falcon-40b-v9":805.0,"openchat8192-13b":805.0,"wizardlm-13b":805.0,"vicuna-13b":805.0,"merlinite-7B-AOT":805.0,"gpt4_0314":805.0,"gpt4_0613_concise":805.0,"jina-chat":805.0,"Contextual-KTO-Mistral-PairRM":805.0,"xwinlm-13b-v0.1":805.0,"LMCocktail-10.7B-v1":805.0,"SPPO-Mistral7B-PairRM-ExPO":805.0,"Mixtral-8x7B-Instruct-v0.1_concise":805.0,"gpt4_1106_preview_concise":805.0,"Mistral-7B-ReMax-v0.1":805.0,"Llama-3-Instruct-8B-SimPO-ExPO":805.0,"dolphin-2.2.1-mistral-7b":805.0,"humpback-llama2-70b":805.0,"openpipe-moa-gpt-4-turbo-v1":805.0,"vicuna-7b-v1.5":805.0,"Starling-LM-7B-alpha":805.0,"falcon-40b-instruct":805.0,"Samba-CoE-v0.2-best-of-16":805.0,"opencoderplus-15b":805.0,"xwinlm-70b-v0.1":805.0,"wizardlm-13b-v1.2":805.0,"aligner-2b_qwen1.5-72b-chat":805.0,"internlm2-chat-7b-ExPO":805.0,"claude-2.1":805.0,"vicuna-7b-v1.3":805.0,"oasst-rlhf-llama-33b":805.0,"zephyr-7b-alpha-ExPO":805.0,"openchat-v3.1-13b":805.0,"SPPO-Llama-3-Instruct-8B-PairRM":805.0,"minotaur-13b":804.0,"tulu-2-dpo-13b-ExPO":805.0,"zephyr-7b-beta-ExPO":805.0,"tulu-2-dpo-7b-ExPO":805.0,"Llama-3-Instruct-8B-SimPO":805.0,"baize-v2-13b":805.0,"guanaco-7b":805.0,"ultralm-13b-v2.0-best-of-16":805.0,"claude-2.1_concise":805.0,"openchat-13b":805.0,"tulu-2-dpo-70b":805.0,"deepseek-llm-67b-chat":805.0,"humpback-llama-65b":805.0,"tulu-2-dpo-70b-ExPO":805.0,"TempNet-LLaMA2-Chat-7B-v0.1":805.0,"nous-hermes-13b":805.0,"gpt-3.5-turbo-0613":805.0,"alpaca-7b_concise":804.0,"baichuan-13b-chat":805.0,"claude-3-5-sonnet-20240620":805.0,"gpt-3.5-turbo-1106":805.0,"minichat-3b":805.0,"Storm-7B":805.0,"oasst-sft-pythia-12b":805.0,"Conifer-7B-DPO":805.0,"Snorkel-Mistral-PairRM-DPO":804.0,"internlm2-chat-20b-ExPO":805.0,"Samba-CoE-v0.2":805.0,"gemini-pro":805.0,"pairrm-tulu-2-70b":805.0,"text_davinci_003":805.0,"gpt4":805.0,"Yi-34B-Chat":805.0,"Starling-LM-7B-beta-ExPO":805.0,"pairrm-Yi-34B-Chat":805.0,"gpt4_1106_preview":805.0,"evo-7b":805.0,"zephyr-7b-beta":805.0,"guanaco-13b":805.0,"alpaca-7b":805.0,"internlm2-chat-20b-ppo":805.0,"gemma-2b-it":805.0,"pairrm-zephyr-7b-beta":805.0,"evo-v2-7b":805.0,"causallm-14b":805.0,"SPPO-Mistral7B-PairRM":805.0,"gpt-3.5-turbo-1106_concise":805.0,"openbuddy-llama-65b-v8":805.0,"claude2-alpaca-13b":805.0,"Starling-LM-7B-alpha-ExPO":805.0,"openbuddy-falcon-7b-v6":805.0,"gemma-7b-it":805.0,"phi-2-sft":805.0,"gpt4_gamed":805.0,"llama-2-70b-chat-hf":804.0,"openbuddy-llama2-70b-v10.1":805.0,"wizardlm-70b":805.0,"ultralm-13b-best-of-16":805.0},"discrete_win_rate":{"gpt-3.5-turbo-0301":8.8819875776,"gpt-3.5-turbo-1106_verbose":11.801242236,"vicuna-13b-v1.5-togetherai":6.8944099379,"Qwen1.5-1.8B-Chat":3.5447761194,"recycled-wizardlm-7b-v1.0":6.5838509317,"aligner-2b_claude-3-opus-20240229":34.4720496894,"Qwen1.5-110B-Chat":31.9875776398,"claude-3-opus-20240229":27.8881987578,"llama-2-7b-chat-hf":4.7826086957,"mistral-medium":20.4968944099,"vicuna-33b-v1.3":11.4285714286,"cohere":11.9254658385,"claude-2":16.3354037267,"guanaco-65b":6.7080745342,"Mixtral-8x7B-Instruct-v0.1":16.8944099379,"openchat-v2-w-13b":8.4472049689,"falcon-7b-instruct":2.1118012422,"wizardlm-13b-v1.1":10.0,"Meta-Llama-3-8B-Instruct":22.049689441,"FsfairX-Zephyr-Chat-v0.1":35.5900621118,"Infinity-Instruct-3M-0613-Mistral-7B":14.6583850932,"Qwen1.5-72B-Chat":25.2173913043,"xwinlm-7b-v0.1":9.6273291925,"Mixtral-8x22B-Instruct-v0.1":21.801242236,"vicuna-13b-v1.5":6.2111801242,"dbrx-instruct":18.3229813665,"zephyr-7b-alpha":7.3913043478,"tulu-2-dpo-13b":9.4409937888,"Qwen1.5-7B-Chat":10.1863354037,"Together-MoA-Lite":56.7701863354,"cut-13b":10.3726708075,"Meta-Llama-3-70B-Instruct":33.1677018634,"vicuna-13b-v1.3":6.4596273292,"claude-instant-1.2":15.0931677019,"airoboros-65b":8.5093167702,"openbuddy-llama2-13b-v11.1":5.3416149068,"phi-2":2.0547945205,"Together-MoA":60.9316770186,"mistral-large-2402":20.6832298137,"openbuddy-llama-30b-v7.1":6.0248447205,"TempNet-LLaMA2-Chat-70B-v0.1":13.9303482587,"pairrm-tulu-2-13b":13.7267080745,"recycled-wizardlm-7b-v2.0":6.2111801242,"Storm-7B-best-of-64":64.4720496894,"vicuna-7b":3.602484472,"claude-3-sonnet-20240229":24.2236024845,"Mistral-7B-Instruct-v0.2":14.099378882,"Samba-CoE-v0.1":15.4658385093,"claude":16.0248447205,"Nanbeige2-8B-Chat":40.248447205,"REBEL-Llama-3-8B-Instruct":33.2919254658,"chatglm2-6b":2.6708074534,"gpt-4o-2024-05-13":53.7267080745,"gpt4_1106_preview_verbose":65.9627329193,"TempNet-LLaMA2-Chat-13B-v0.1":6.9565217391,"text_davinci_001":3.0510585305,"Mixtral-8x7B-Instruct-v0.1_verbose":24.2236024845,"baize-v2-7b":3.2298136646,"phi-2-dpo":7.0807453416,"alpaca-farm-ppo-human":4.1614906832,"Nanbeige2-16B-Chat":35.9627329193,"gpt4_0613":14.7826086957,"pythia-12b-mix-sft":2.3602484472,"alpaca-7b-neft":2.7329192547,"Qwen1.5-14B-Chat":17.2670807453,"gpt-4-0125-preview":56.149068323,"guanaco-33b":4.5962732919,"oasst-sft-llama-33b":4.7826086957,"gpt4_0613_verbose":21.4906832298,"llama-2-chat-7b-evol70k-neft":7.0807453416,"gpt35_turbo_instruct":8.3955223881,"platolm-7b":5.3549190535,"llama-2-13b-chat-hf":7.5155279503,"Nanbeige-Plus-Chat-v0.1":56.7701863354,"openchat-v2-13b":7.1428571429,"mistral-orpo-beta":11.9875776398,"Snorkel-Mistral-PairRM-DPO-best-of-16":33.6645962733,"tulu-2-dpo-7b":8.0124223602,"alpaca-7b_verbose":2.8678304239,"OpenHermes-2.5-Mistral-7B":9.5031055901,"claude-2.1_verbose":23.7888198758,"ultralm-13b-v2.0":6.3354037267,"deita-7b-v1.0":11.9875776398,"minichat-1.5-3b":5.9627329193,"Qwen-14B-Chat":7.4534161491,"airoboros-33b":8.0124223602,"alpaca-farm-ppo-sim-gpt4-20k":3.4161490683,"ultralm-13b":4.8447204969,"openbuddy-falcon-40b-v9":5.7142857143,"openchat8192-13b":6.3354037267,"wizardlm-13b":5.4658385093,"vicuna-13b":5.5900621118,"merlinite-7B-AOT":29.0683229814,"gpt4_0314":21.7391304348,"gpt4_0613_concise":9.1304347826,"jina-chat":7.5155279503,"Contextual-KTO-Mistral-PairRM":32.3602484472,"xwinlm-13b-v0.1":16.2732919255,"LMCocktail-10.7B-v1":12.9813664596,"SPPO-Mistral7B-PairRM-ExPO":34.0372670807,"Mixtral-8x7B-Instruct-v0.1_concise":13.0434782609,"gpt4_1106_preview_concise":22.049689441,"Mistral-7B-ReMax-v0.1":15.0310559006,"Llama-3-Instruct-8B-SimPO-ExPO":40.4347826087,"dolphin-2.2.1-mistral-7b":8.6335403727,"humpback-llama2-70b":9.6273291925,"openpipe-moa-gpt-4-turbo-v1":64.4099378882,"vicuna-7b-v1.5":4.5341614907,"Starling-LM-7B-alpha":12.7329192547,"falcon-40b-instruct":3.4161490683,"Samba-CoE-v0.2-best-of-16":25.1552795031,"opencoderplus-15b":6.6459627329,"xwinlm-70b-v0.1":20.8695652174,"wizardlm-13b-v1.2":10.3726708075,"aligner-2b_qwen1.5-72b-chat":31.801242236,"internlm2-chat-7b-ExPO":26.0248447205,"claude-2.1":14.4099378882,"vicuna-7b-v1.3":4.0372670807,"oasst-rlhf-llama-33b":5.5900621118,"zephyr-7b-alpha-ExPO":9.8757763975,"openchat-v3.1-13b":10.248447205,"SPPO-Llama-3-Instruct-8B-PairRM":38.5714285714,"minotaur-13b":5.4726368159,"tulu-2-dpo-13b-ExPO":15.3416149068,"zephyr-7b-beta-ExPO":11.0559006211,"tulu-2-dpo-7b-ExPO":11.3043478261,"Llama-3-Instruct-8B-SimPO":39.6894409938,"baize-v2-13b":4.1614906832,"guanaco-7b":2.6708074534,"ultralm-13b-v2.0-best-of-16":12.298136646,"claude-2.1_concise":9.1304347826,"openchat-13b":7.2670807453,"tulu-2-dpo-70b":14.9689440994,"deepseek-llm-67b-chat":11.3043478261,"humpback-llama-65b":8.7577639752,"tulu-2-dpo-70b-ExPO":22.9192546584,"TempNet-LLaMA2-Chat-7B-v0.1":4.9068322981,"nous-hermes-13b":5.4037267081,"gpt-3.5-turbo-0613":12.6708074534,"alpaca-7b_concise":1.9900497512,"baichuan-13b-chat":1.801242236,"claude-3-5-sonnet-20240620":38.7577639752,"gpt-3.5-turbo-1106":8.198757764,"minichat-3b":2.9813664596,"Storm-7B":49.3167701863,"oasst-sft-pythia-12b":1.7391304348,"Conifer-7B-DPO":10.8695652174,"Snorkel-Mistral-PairRM-DPO":28.7935323383,"internlm2-chat-20b-ExPO":46.5838509317,"Samba-CoE-v0.2":19.8136645963,"gemini-pro":17.0807453416,"pairrm-tulu-2-70b":17.3913043478,"text_davinci_003":1.9875776398,"gpt4":22.7329192547,"Yi-34B-Chat":27.4534161491,"Starling-LM-7B-beta-ExPO":27.950310559,"pairrm-Yi-34B-Chat":29.8757763975,"gpt4_1106_preview":50.0,"evo-7b":14.1614906832,"zephyr-7b-beta":9.8136645963,"guanaco-13b":2.9192546584,"alpaca-7b":2.298136646,"internlm2-chat-20b-ppo":21.3043478261,"gemma-2b-it":2.8571428571,"pairrm-zephyr-7b-beta":12.2360248447,"evo-v2-7b":19.8136645963,"causallm-14b":10.3105590062,"SPPO-Mistral7B-PairRM":30.9316770186,"gpt-3.5-turbo-1106_concise":7.3291925466,"openbuddy-llama-65b-v8":8.1366459627,"claude2-alpaca-13b":7.3291925466,"Starling-LM-7B-alpha-ExPO":18.3850931677,"openbuddy-falcon-7b-v6":3.3540372671,"gemma-7b-it":6.2732919255,"phi-2-sft":3.4782608696,"gpt4_gamed":4.099378882,"llama-2-70b-chat-hf":12.9353233831,"openbuddy-llama2-70b-v10.1":7.3291925466,"wizardlm-70b":13.2919254658,"ultralm-13b-best-of-16":10.0621118012},"length_controlled_winrate":{"gpt-3.5-turbo-0301":18.093241552,"gpt-3.5-turbo-1106_verbose":22.0009370217,"vicuna-13b-v1.5-togetherai":11.6853569655,"Qwen1.5-1.8B-Chat":2.5884988492,"recycled-wizardlm-7b-v1.0":6.901477322,"aligner-2b_claude-3-opus-20240229":41.8230717152,"Qwen1.5-110B-Chat":43.905552211,"claude-3-opus-20240229":40.5095080124,"llama-2-7b-chat-hf":5.3548212795,"mistral-medium":28.6143374017,"vicuna-33b-v1.3":17.5745753109,"cohere":10.8930208866,"claude-2":28.1551961416,"guanaco-65b":8.2529169916,"Mixtral-8x7B-Instruct-v0.1":23.6884826013,"openchat-v2-w-13b":12.030427771,"falcon-7b-instruct":4.0369375668,"wizardlm-13b-v1.1":13.9157205928,"Meta-Llama-3-8B-Instruct":22.9187846731,"FsfairX-Zephyr-Chat-v0.1":34.787447623,"Infinity-Instruct-3M-0613-Mistral-7B":25.5015577947,"Qwen1.5-72B-Chat":36.571754112,"xwinlm-7b-v0.1":10.8122056273,"Mixtral-8x22B-Instruct-v0.1":30.8788102941,"vicuna-13b-v1.5":10.4844382985,"dbrx-instruct":25.1853410397,"zephyr-7b-alpha":10.2897608887,"tulu-2-dpo-13b":11.5544794281,"Qwen1.5-7B-Chat":14.7484310443,"Together-MoA-Lite":59.1415240989,"cut-13b":12.1547817539,"Meta-Llama-3-70B-Instruct":34.4245971745,"vicuna-13b-v1.3":10.8431649437,"claude-instant-1.2":25.6122590254,"airoboros-65b":11.0076424064,"openbuddy-llama2-13b-v11.1":9.159089775,"phi-2":4.3986822709,"Together-MoA":65.3799697685,"mistral-large-2402":32.6520799853,"openbuddy-llama-30b-v7.1":10.2144949912,"TempNet-LLaMA2-Chat-70B-v0.1":15.8311627784,"pairrm-tulu-2-13b":17.405203698,"recycled-wizardlm-7b-v2.0":7.5216099553,"Storm-7B-best-of-64":61.637895572,"vicuna-7b":6.2772177385,"claude-3-sonnet-20240229":34.8724743624,"Mistral-7B-Instruct-v0.2":17.111251846,"Samba-CoE-v0.1":22.8658373348,"claude":27.2895044437,"Nanbeige2-8B-Chat":25.2417704867,"REBEL-Llama-3-8B-Instruct":31.4699427971,"chatglm2-6b":4.3592829268,"gpt-4o-2024-05-13":57.4568288333,"gpt4_1106_preview_verbose":51.5750079797,"TempNet-LLaMA2-Chat-13B-v0.1":8.5783553109,"text_davinci_001":9.0257288521,"Mixtral-8x7B-Instruct-v0.1_verbose":23.2231207809,"baize-v2-7b":4.382564905,"phi-2-dpo":7.7708946203,"alpaca-farm-ppo-human":6.4186032949,"Nanbeige2-16B-Chat":40.5912863493,"gpt4_0613":30.1833223167,"pythia-12b-mix-sft":4.2213618614,"alpaca-7b-neft":3.5091458375,"Qwen1.5-14B-Chat":23.8966467702,"gpt-4-0125-preview":56.3562938462,"guanaco-33b":5.6900190909,"oasst-sft-llama-33b":9.8664121438,"gpt4_0613_verbose":33.8212668866,"llama-2-chat-7b-evol70k-neft":7.5330526555,"gpt35_turbo_instruct":17.7278010829,"platolm-7b":10.5434020728,"llama-2-13b-chat-hf":8.4360145489,"Nanbeige-Plus-Chat-v0.1":44.4596624034,"openchat-v2-13b":10.3996073385,"mistral-orpo-beta":14.7167494307,"Snorkel-Mistral-PairRM-DPO-best-of-16":29.9743216131,"tulu-2-dpo-7b":9.2002656115,"alpaca-7b_verbose":6.8163068164,"OpenHermes-2.5-Mistral-7B":16.2485776967,"claude-2.1_verbose":30.2911791666,"ultralm-13b-v2.0":9.1290184442,"deita-7b-v1.0":16.0590135397,"minichat-1.5-3b":7.7016328215,"Qwen-14B-Chat":12.3787417907,"airoboros-33b":10.7190026781,"alpaca-farm-ppo-sim-gpt4-20k":7.1218081016,"ultralm-13b":7.1081913613,"openbuddy-falcon-40b-v9":8.9889364779,"openchat8192-13b":7.8970617346,"wizardlm-13b":9.8281507688,"vicuna-13b":9.2220600237,"merlinite-7B-AOT":31.721885287,"gpt4_0314":35.3070612164,"gpt4_0613_concise":21.5779909145,"jina-chat":15.8660040495,"Contextual-KTO-Mistral-PairRM":29.7058089397,"xwinlm-13b-v0.1":17.9189378982,"LMCocktail-10.7B-v1":18.9507103867,"SPPO-Mistral7B-PairRM-ExPO":31.9003876312,"Mixtral-8x7B-Instruct-v0.1_concise":22.9626094728,"gpt4_1106_preview_concise":41.8966015912,"Mistral-7B-ReMax-v0.1":20.5513677023,"Llama-3-Instruct-8B-SimPO-ExPO":45.807978034,"dolphin-2.2.1-mistral-7b":13.1214776504,"humpback-llama2-70b":16.2491642314,"openpipe-moa-gpt-4-turbo-v1":68.3786625033,"vicuna-7b-v1.5":7.6168927319,"Starling-LM-7B-alpha":14.6904710794,"falcon-40b-instruct":5.6075325447,"Samba-CoE-v0.2-best-of-16":31.5065442681,"opencoderplus-15b":8.1524101557,"xwinlm-70b-v0.1":24.6496860571,"wizardlm-13b-v1.2":14.4625906943,"aligner-2b_qwen1.5-72b-chat":36.7258688784,"internlm2-chat-7b-ExPO":22.6674802488,"claude-2.1":25.2519438861,"vicuna-7b-v1.3":7.1564609564,"oasst-rlhf-llama-33b":7.9709218373,"zephyr-7b-alpha-ExPO":13.6232522647,"openchat-v3.1-13b":14.5033879568,"SPPO-Llama-3-Instruct-8B-PairRM":38.5628066368,"minotaur-13b":11.4652513168,"tulu-2-dpo-13b-ExPO":17.6509979624,"zephyr-7b-beta-ExPO":14.0012119801,"tulu-2-dpo-7b-ExPO":11.6088057579,"Llama-3-Instruct-8B-SimPO":44.6804680926,"baize-v2-13b":7.012247205,"guanaco-7b":2.8711168131,"ultralm-13b-v2.0-best-of-16":14.1989875666,"claude-2.1_concise":18.2084579084,"openchat-13b":8.8060534912,"tulu-2-dpo-70b":21.2386100384,"deepseek-llm-67b-chat":17.8433840899,"humpback-llama-65b":12.7998599959,"tulu-2-dpo-70b-ExPO":25.7233081711,"TempNet-LLaMA2-Chat-7B-v0.1":5.7396138367,"nous-hermes-13b":9.7178634178,"gpt-3.5-turbo-0613":22.3525129805,"alpaca-7b_concise":4.4672516799,"baichuan-13b-chat":2.0621702536,"claude-3-5-sonnet-20240620":52.3667542714,"gpt-3.5-turbo-1106":19.300589035,"minichat-3b":5.7293328759,"Storm-7B":50.4080792281,"oasst-sft-pythia-12b":3.2701021145,"Conifer-7B-DPO":17.1124958828,"Snorkel-Mistral-PairRM-DPO":26.3914464573,"internlm2-chat-20b-ExPO":27.2257594808,"Samba-CoE-v0.2":27.6242673501,"gemini-pro":24.381776108,"pairrm-tulu-2-70b":21.4284039755,"text_davinci_003":4.5664105675,"gpt4":38.1280897444,"Yi-34B-Chat":27.1905478776,"Starling-LM-7B-beta-ExPO":26.4869564984,"pairrm-Yi-34B-Chat":28.8148408668,"gpt4_1106_preview":50.0,"evo-7b":16.4893860042,"zephyr-7b-beta":13.2031984931,"guanaco-13b":3.0037873296,"alpaca-7b":5.8754871633,"internlm2-chat-20b-ppo":18.7487394854,"gemma-2b-it":5.4374536204,"pairrm-zephyr-7b-beta":15.529867295,"evo-v2-7b":23.357705702,"causallm-14b":15.720325189,"SPPO-Mistral7B-PairRM":30.4941379652,"gpt-3.5-turbo-1106_concise":15.7695209839,"openbuddy-llama-65b-v8":12.4693562891,"claude2-alpaca-13b":11.4988982132,"Starling-LM-7B-alpha-ExPO":19.4741654606,"openbuddy-falcon-7b-v6":4.8261244822,"gemma-7b-it":10.4257604037,"phi-2-sft":5.8537876906,"gpt4_gamed":12.1887640576,"llama-2-70b-chat-hf":14.6896485884,"openbuddy-llama2-70b-v10.1":12.5721732723,"wizardlm-70b":17.5750607375,"ultralm-13b-best-of-16":9.8760888169},"lc_standard_error":{"gpt-3.5-turbo-0301":0.7864976807,"gpt-3.5-turbo-1106_verbose":0.8544953416,"vicuna-13b-v1.5-togetherai":0.6243797898,"Qwen1.5-1.8B-Chat":0.2021610274,"recycled-wizardlm-7b-v1.0":0.4105893841,"aligner-2b_claude-3-opus-20240229":0.7776876699,"Qwen1.5-110B-Chat":0.8945807936,"claude-3-opus-20240229":0.8837504763,"llama-2-7b-chat-hf":0.3326400931,"mistral-medium":0.9075464438,"vicuna-33b-v1.3":0.7099362877,"cohere":0.5206791146,"claude-2":0.8779084794,"guanaco-65b":0.46281361,"Mixtral-8x7B-Instruct-v0.1":0.9011105015,"openchat-v2-w-13b":0.5657607148,"falcon-7b-instruct":0.268726544,"wizardlm-13b-v1.1":0.6712555976,"Meta-Llama-3-8B-Instruct":0.849800882,"FsfairX-Zephyr-Chat-v0.1":0.7594505141,"Infinity-Instruct-3M-0613-Mistral-7B":0.7760697229,"Qwen1.5-72B-Chat":0.9357421321,"xwinlm-7b-v0.1":0.5519849159,"Mixtral-8x22B-Instruct-v0.1":0.9518125819,"vicuna-13b-v1.5":0.5980193852,"dbrx-instruct":0.8999456518,"zephyr-7b-alpha":0.5879820221,"tulu-2-dpo-13b":0.6494943093,"Qwen1.5-7B-Chat":0.6490365375,"Together-MoA-Lite":0.7580510219,"cut-13b":0.6383138465,"Meta-Llama-3-70B-Instruct":0.8691832384,"vicuna-13b-v1.3":0.6100962742,"claude-instant-1.2":0.87464248,"airoboros-65b":0.6004520879,"openbuddy-llama2-13b-v11.1":0.5636847159,"phi-2":0.1293627793,"Together-MoA":0.7392392837,"mistral-large-2402":0.9044632955,"openbuddy-llama-30b-v7.1":0.6099418552,"TempNet-LLaMA2-Chat-70B-v0.1":0.7195404924,"pairrm-tulu-2-13b":0.7958946232,"recycled-wizardlm-7b-v2.0":0.4355543699,"Storm-7B-best-of-64":0.6799412402,"vicuna-7b":0.3964740967,"claude-3-sonnet-20240229":0.949844689,"Mistral-7B-Instruct-v0.2":0.7875592102,"Samba-CoE-v0.1":0.7405123259,"claude":0.858614564,"Nanbeige2-8B-Chat":0.5909370499,"REBEL-Llama-3-8B-Instruct":0.8138922262,"chatglm2-6b":0.2913010016,"gpt-4o-2024-05-13":0.7774399385,"gpt4_1106_preview_verbose":0.8313707608,"TempNet-LLaMA2-Chat-13B-v0.1":0.4783538284,"text_davinci_001":0.2169278281,"Mixtral-8x7B-Instruct-v0.1_verbose":0.7975932103,"baize-v2-7b":0.3307775329,"phi-2-dpo":0.420015191,"alpaca-farm-ppo-human":0.4202234849,"Nanbeige2-16B-Chat":0.8504106275,"gpt4_0613":0.7874508454,"pythia-12b-mix-sft":0.2932467883,"alpaca-7b-neft":0.2516233369,"Qwen1.5-14B-Chat":0.7729838609,"gpt-4-0125-preview":0.7731843456,"guanaco-33b":0.3195322556,"oasst-sft-llama-33b":0.5204539206,"gpt4_0613_verbose":0.8842151461,"llama-2-chat-7b-evol70k-neft":0.4277221418,"gpt35_turbo_instruct":0.3748783811,"platolm-7b":0.3937696385,"llama-2-13b-chat-hf":0.5161956367,"Nanbeige-Plus-Chat-v0.1":0.7209678864,"openchat-v2-13b":0.5398936504,"mistral-orpo-beta":0.6895695724,"Snorkel-Mistral-PairRM-DPO-best-of-16":0.7464891533,"tulu-2-dpo-7b":0.5465634637,"alpaca-7b_verbose":0.2437107339,"OpenHermes-2.5-Mistral-7B":0.7206735233,"claude-2.1_verbose":0.6612722747,"ultralm-13b-v2.0":0.5248779977,"deita-7b-v1.0":0.7398615266,"minichat-1.5-3b":0.4364271175,"Qwen-14B-Chat":0.6714412819,"airoboros-33b":0.5566576337,"alpaca-farm-ppo-sim-gpt4-20k":0.456168214,"ultralm-13b":0.4337345632,"openbuddy-falcon-40b-v9":0.545771106,"openchat8192-13b":0.4356316711,"wizardlm-13b":0.5385026234,"vicuna-13b":0.5388256266,"merlinite-7B-AOT":0.8150560619,"gpt4_0314":0.8997916758,"gpt4_0613_concise":0.7524372534,"jina-chat":0.6805565304,"Contextual-KTO-Mistral-PairRM":0.7122554396,"xwinlm-13b-v0.1":0.7513299972,"LMCocktail-10.7B-v1":0.8369176162,"SPPO-Mistral7B-PairRM-ExPO":0.7655500294,"Mixtral-8x7B-Instruct-v0.1_concise":0.8710401023,"gpt4_1106_preview_concise":0.7406558917,"Mistral-7B-ReMax-v0.1":0.807838924,"Llama-3-Instruct-8B-SimPO-ExPO":0.8703329817,"dolphin-2.2.1-mistral-7b":0.6251596825,"humpback-llama2-70b":0.6984941388,"openpipe-moa-gpt-4-turbo-v1":0.7309418615,"vicuna-7b-v1.5":0.4868743581,"Starling-LM-7B-alpha":0.658381614,"falcon-40b-instruct":0.3565968022,"Samba-CoE-v0.2-best-of-16":0.7338723477,"opencoderplus-15b":0.4567320517,"xwinlm-70b-v0.1":0.9059240217,"wizardlm-13b-v1.2":0.6741078562,"aligner-2b_qwen1.5-72b-chat":0.6787999003,"internlm2-chat-7b-ExPO":0.629923982,"claude-2.1":0.7515108894,"vicuna-7b-v1.3":0.4355620786,"oasst-rlhf-llama-33b":0.4061516205,"zephyr-7b-alpha-ExPO":0.7160268998,"openchat-v3.1-13b":0.6974328561,"SPPO-Llama-3-Instruct-8B-PairRM":0.8694594533,"minotaur-13b":0.368757115,"tulu-2-dpo-13b-ExPO":0.5166082438,"zephyr-7b-beta-ExPO":0.5303710259,"tulu-2-dpo-7b-ExPO":0.4355576278,"Llama-3-Instruct-8B-SimPO":0.8789917177,"baize-v2-13b":0.4685705196,"guanaco-7b":0.2018869696,"ultralm-13b-v2.0-best-of-16":0.6555243163,"claude-2.1_concise":0.6338526283,"openchat-13b":0.4470052867,"tulu-2-dpo-70b":0.8610574163,"deepseek-llm-67b-chat":0.7439504148,"humpback-llama-65b":0.6567402094,"tulu-2-dpo-70b-ExPO":0.4593179402,"TempNet-LLaMA2-Chat-7B-v0.1":0.3407340673,"nous-hermes-13b":0.5572824918,"gpt-3.5-turbo-0613":0.8045156377,"alpaca-7b_concise":0.2820018938,"baichuan-13b-chat":0.1525670221,"claude-3-5-sonnet-20240620":0.7976856335,"gpt-3.5-turbo-1106":0.7682908268,"minichat-3b":0.3565910812,"Storm-7B":0.7188927916,"oasst-sft-pythia-12b":0.2064079261,"Conifer-7B-DPO":0.7602280224,"Snorkel-Mistral-PairRM-DPO":0.6739888325,"internlm2-chat-20b-ExPO":0.5877331102,"Samba-CoE-v0.2":0.6875926799,"gemini-pro":0.8158961767,"pairrm-tulu-2-70b":0.8359305763,"text_davinci_003":0.3109387936,"gpt4":0.9069675584,"Yi-34B-Chat":0.7470363322,"Starling-LM-7B-beta-ExPO":0.7549415682,"pairrm-Yi-34B-Chat":0.8310750322,"gpt4_1106_preview":0.0,"evo-7b":0.502828858,"zephyr-7b-beta":0.6521227924,"guanaco-13b":0.2069624951,"alpaca-7b":0.3755224975,"internlm2-chat-20b-ppo":0.7522583795,"gemma-2b-it":0.3236386036,"pairrm-zephyr-7b-beta":0.7455357676,"evo-v2-7b":0.6353106561,"causallm-14b":0.7103430968,"SPPO-Mistral7B-PairRM":0.8458266977,"gpt-3.5-turbo-1106_concise":0.7318554971,"openbuddy-llama-65b-v8":0.6457736922,"claude2-alpaca-13b":0.6646440129,"Starling-LM-7B-alpha-ExPO":0.4701002864,"openbuddy-falcon-7b-v6":0.3350353845,"gemma-7b-it":0.4807679381,"phi-2-sft":0.3931141644,"gpt4_gamed":0.3987510662,"llama-2-70b-chat-hf":0.6625475757,"openbuddy-llama2-70b-v10.1":0.6740810303,"wizardlm-70b":0.7233004015,"ultralm-13b-best-of-16":0.4814959281},"num_tokens_mean":{"gpt-3.5-turbo-0301":179,"gpt-3.5-turbo-1106_verbose":214,"vicuna-13b-v1.5-togetherai":231,"Qwen1.5-1.8B-Chat":586,"recycled-wizardlm-7b-v1.0":299,"aligner-2b_claude-3-opus-20240229":354,"Qwen1.5-110B-Chat":346,"claude-3-opus-20240229":292,"llama-2-7b-chat-hf":302,"mistral-medium":327,"vicuna-33b-v1.3":315,"cohere":396,"claude-2":227,"guanaco-65b":262,"Mixtral-8x7B-Instruct-v0.1":311,"openchat-v2-w-13b":335,"falcon-7b-instruct":106,"wizardlm-13b-v1.1":325,"Meta-Llama-3-8B-Instruct":412,"FsfairX-Zephyr-Chat-v0.1":505,"Infinity-Instruct-3M-0613-Mistral-7B":106,"Qwen1.5-72B-Chat":342,"xwinlm-7b-v0.1":407,"Mixtral-8x22B-Instruct-v0.1":307,"vicuna-13b-v1.5":229,"dbrx-instruct":310,"zephyr-7b-alpha":283,"tulu-2-dpo-13b":370,"Qwen1.5-7B-Chat":342,"Together-MoA-Lite":420,"cut-13b":348,"Meta-Llama-3-70B-Instruct":416,"vicuna-13b-v1.3":244,"claude-instant-1.2":233,"airoboros-65b":312,"openbuddy-llama2-13b-v11.1":228,"phi-2":142,"Together-MoA":386,"mistral-large-2402":290,"openbuddy-llama-30b-v7.1":208,"TempNet-LLaMA2-Chat-70B-v0.1":384,"pairrm-tulu-2-13b":313,"recycled-wizardlm-7b-v2.0":317,"Storm-7B-best-of-64":486,"vicuna-7b":224,"claude-3-sonnet-20240229":297,"Mistral-7B-Instruct-v0.2":362,"Samba-CoE-v0.1":258,"claude":229,"Nanbeige2-8B-Chat":561,"REBEL-Llama-3-8B-Instruct":509,"chatglm2-6b":234,"gpt-4o-2024-05-13":406,"gpt4_1106_preview_verbose":505,"TempNet-LLaMA2-Chat-13B-v0.1":320,"text_davinci_001":67,"Mixtral-8x7B-Instruct-v0.1_verbose":438,"baize-v2-7b":247,"phi-2-dpo":356,"alpaca-farm-ppo-human":169,"Nanbeige2-16B-Chat":394,"gpt4_0613":245,"pythia-12b-mix-sft":197,"alpaca-7b-neft":216,"Qwen1.5-14B-Chat":348,"gpt-4-0125-preview":417,"guanaco-33b":283,"oasst-sft-llama-33b":161,"gpt4_0613_verbose":313,"llama-2-chat-7b-evol70k-neft":330,"gpt35_turbo_instruct":219,"platolm-7b":272,"llama-2-13b-chat-hf":316,"Nanbeige-Plus-Chat-v0.1":529,"openchat-v2-13b":330,"mistral-orpo-beta":344,"Snorkel-Mistral-PairRM-DPO-best-of-16":526,"tulu-2-dpo-7b":399,"alpaca-7b_verbose":119,"OpenHermes-2.5-Mistral-7B":237,"claude-2.1_verbose":292,"ultralm-13b-v2.0":299,"deita-7b-v1.0":303,"minichat-1.5-3b":322,"Qwen-14B-Chat":223,"airoboros-33b":315,"alpaca-farm-ppo-sim-gpt4-20k":104,"ultralm-13b":233,"openbuddy-falcon-40b-v9":238,"openchat8192-13b":359,"wizardlm-13b":211,"vicuna-13b":224,"merlinite-7B-AOT":385,"gpt4_0314":289,"gpt4_0613_concise":138,"jina-chat":138,"Contextual-KTO-Mistral-PairRM":506,"xwinlm-13b-v0.1":400,"LMCocktail-10.7B-v1":244,"SPPO-Mistral7B-PairRM-ExPO":459,"Mixtral-8x7B-Instruct-v0.1_concise":195,"gpt4_1106_preview_concise":244,"Mistral-7B-ReMax-v0.1":314,"Llama-3-Instruct-8B-SimPO-ExPO":370,"dolphin-2.2.1-mistral-7b":243,"humpback-llama2-70b":234,"openpipe-moa-gpt-4-turbo-v1":377,"vicuna-7b-v1.5":234,"Starling-LM-7B-alpha":392,"falcon-40b-instruct":148,"Samba-CoE-v0.2-best-of-16":312,"opencoderplus-15b":354,"xwinlm-70b-v0.1":381,"wizardlm-13b-v1.2":354,"aligner-2b_qwen1.5-72b-chat":383,"internlm2-chat-7b-ExPO":494,"claude-2.1":228,"vicuna-7b-v1.3":238,"oasst-rlhf-llama-33b":243,"zephyr-7b-alpha-ExPO":267,"openchat-v3.1-13b":318,"SPPO-Llama-3-Instruct-8B-PairRM":443,"minotaur-13b":186,"tulu-2-dpo-13b-ExPO":350,"zephyr-7b-beta-ExPO":297,"tulu-2-dpo-7b-ExPO":373,"Llama-3-Instruct-8B-SimPO":385,"baize-v2-13b":201,"guanaco-7b":311,"ultralm-13b-v2.0-best-of-16":354,"claude-2.1_concise":119,"openchat-13b":349,"tulu-2-dpo-70b":313,"deepseek-llm-67b-chat":250,"humpback-llama-65b":254,"tulu-2-dpo-70b-ExPO":368,"TempNet-LLaMA2-Chat-7B-v0.1":308,"nous-hermes-13b":187,"gpt-3.5-turbo-0613":270,"alpaca-7b_concise":77,"baichuan-13b-chat":310,"claude-3-5-sonnet-20240620":315,"gpt-3.5-turbo-1106":164,"minichat-3b":186,"Storm-7B":429,"oasst-sft-pythia-12b":158,"Conifer-7B-DPO":266,"Snorkel-Mistral-PairRM-DPO":566,"internlm2-chat-20b-ExPO":693,"Samba-CoE-v0.2":292,"gemini-pro":313,"pairrm-tulu-2-70b":405,"text_davinci_003":69,"gpt4":287,"Yi-34B-Chat":447,"Starling-LM-7B-beta-ExPO":447,"pairrm-Yi-34B-Chat":464,"gpt4_1106_preview":431,"evo-7b":382,"zephyr-7b-beta":311,"guanaco-13b":416,"alpaca-7b":85,"internlm2-chat-20b-ppo":513,"gemma-2b-it":222,"pairrm-zephyr-7b-beta":315,"evo-v2-7b":375,"causallm-14b":302,"SPPO-Mistral7B-PairRM":431,"gpt-3.5-turbo-1106_concise":90,"openbuddy-llama-65b-v8":249,"claude2-alpaca-13b":243,"Starling-LM-7B-alpha-ExPO":376,"openbuddy-falcon-7b-v6":248,"gemma-7b-it":241,"phi-2-sft":225,"gpt4_gamed":15,"llama-2-70b-chat-hf":377,"openbuddy-llama2-70b-v10.1":230,"wizardlm-70b":332,"ultralm-13b-best-of-16":400},"num_tokens_std":{"gpt-3.5-turbo-0301":150,"gpt-3.5-turbo-1106_verbose":144,"vicuna-13b-v1.5-togetherai":163,"Qwen1.5-1.8B-Chat":611,"recycled-wizardlm-7b-v1.0":186,"aligner-2b_claude-3-opus-20240229":206,"Qwen1.5-110B-Chat":211,"claude-3-opus-20240229":156,"llama-2-7b-chat-hf":170,"mistral-medium":305,"vicuna-33b-v1.3":198,"cohere":225,"claude-2":114,"guanaco-65b":212,"Mixtral-8x7B-Instruct-v0.1":219,"openchat-v2-w-13b":210,"falcon-7b-instruct":177,"wizardlm-13b-v1.1":213,"Meta-Llama-3-8B-Instruct":231,"FsfairX-Zephyr-Chat-v0.1":311,"Infinity-Instruct-3M-0613-Mistral-7B":102,"Qwen1.5-72B-Chat":346,"xwinlm-7b-v0.1":308,"Mixtral-8x22B-Instruct-v0.1":253,"vicuna-13b-v1.5":164,"dbrx-instruct":193,"zephyr-7b-alpha":272,"tulu-2-dpo-13b":592,"Qwen1.5-7B-Chat":199,"Together-MoA-Lite":233,"cut-13b":208,"Meta-Llama-3-70B-Instruct":238,"vicuna-13b-v1.3":167,"claude-instant-1.2":132,"airoboros-65b":372,"openbuddy-llama2-13b-v11.1":175,"phi-2":312,"Together-MoA":217,"mistral-large-2402":191,"openbuddy-llama-30b-v7.1":162,"TempNet-LLaMA2-Chat-70B-v0.1":258,"pairrm-tulu-2-13b":194,"recycled-wizardlm-7b-v2.0":173,"Storm-7B-best-of-64":287,"vicuna-7b":161,"claude-3-sonnet-20240229":178,"Mistral-7B-Instruct-v0.2":360,"Samba-CoE-v0.1":194,"claude":123,"Nanbeige2-8B-Chat":239,"REBEL-Llama-3-8B-Instruct":463,"chatglm2-6b":352,"gpt-4o-2024-05-13":273,"gpt4_1106_preview_verbose":261,"TempNet-LLaMA2-Chat-13B-v0.1":178,"text_davinci_001":78,"Mixtral-8x7B-Instruct-v0.1_verbose":225,"baize-v2-7b":183,"phi-2-dpo":188,"alpaca-farm-ppo-human":171,"Nanbeige2-16B-Chat":214,"gpt4_0613":164,"pythia-12b-mix-sft":181,"alpaca-7b-neft":97,"Qwen1.5-14B-Chat":224,"gpt-4-0125-preview":242,"guanaco-33b":236,"oasst-sft-llama-33b":145,"gpt4_0613_verbose":166,"llama-2-chat-7b-evol70k-neft":143,"gpt35_turbo_instruct":241,"platolm-7b":187,"llama-2-13b-chat-hf":175,"Nanbeige-Plus-Chat-v0.1":216,"openchat-v2-13b":205,"mistral-orpo-beta":205,"Snorkel-Mistral-PairRM-DPO-best-of-16":255,"tulu-2-dpo-7b":738,"alpaca-7b_verbose":111,"OpenHermes-2.5-Mistral-7B":225,"claude-2.1_verbose":120,"ultralm-13b-v2.0":188,"deita-7b-v1.0":221,"minichat-1.5-3b":215,"Qwen-14B-Chat":227,"airoboros-33b":324,"alpaca-farm-ppo-sim-gpt4-20k":68,"ultralm-13b":158,"openbuddy-falcon-40b-v9":198,"openchat8192-13b":257,"wizardlm-13b":155,"vicuna-13b":153,"merlinite-7B-AOT":180,"gpt4_0314":212,"gpt4_0613_concise":110,"jina-chat":101,"Contextual-KTO-Mistral-PairRM":272,"xwinlm-13b-v0.1":268,"LMCocktail-10.7B-v1":168,"SPPO-Mistral7B-PairRM-ExPO":193,"Mixtral-8x7B-Instruct-v0.1_concise":185,"gpt4_1106_preview_concise":184,"Mistral-7B-ReMax-v0.1":188,"Llama-3-Instruct-8B-SimPO-ExPO":191,"dolphin-2.2.1-mistral-7b":218,"humpback-llama2-70b":214,"openpipe-moa-gpt-4-turbo-v1":193,"vicuna-7b-v1.5":165,"Starling-LM-7B-alpha":250,"falcon-40b-instruct":182,"Samba-CoE-v0.2-best-of-16":199,"opencoderplus-15b":272,"xwinlm-70b-v0.1":267,"wizardlm-13b-v1.2":308,"aligner-2b_qwen1.5-72b-chat":210,"internlm2-chat-7b-ExPO":232,"claude-2.1":114,"vicuna-7b-v1.3":159,"oasst-rlhf-llama-33b":257,"zephyr-7b-alpha-ExPO":204,"openchat-v3.1-13b":224,"SPPO-Llama-3-Instruct-8B-PairRM":208,"minotaur-13b":164,"tulu-2-dpo-13b-ExPO":203,"zephyr-7b-beta-ExPO":217,"tulu-2-dpo-7b-ExPO":217,"Llama-3-Instruct-8B-SimPO":200,"baize-v2-13b":149,"guanaco-7b":292,"ultralm-13b-v2.0-best-of-16":169,"claude-2.1_concise":88,"openchat-13b":241,"tulu-2-dpo-70b":281,"deepseek-llm-67b-chat":206,"humpback-llama-65b":156,"tulu-2-dpo-70b-ExPO":203,"TempNet-LLaMA2-Chat-7B-v0.1":178,"nous-hermes-13b":201,"gpt-3.5-turbo-0613":194,"alpaca-7b_concise":70,"baichuan-13b-chat":400,"claude-3-5-sonnet-20240620":200,"gpt-3.5-turbo-1106":134,"minichat-3b":156,"Storm-7B":188,"oasst-sft-pythia-12b":186,"Conifer-7B-DPO":163,"Snorkel-Mistral-PairRM-DPO":342,"internlm2-chat-20b-ExPO":283,"Samba-CoE-v0.2":196,"gemini-pro":227,"pairrm-tulu-2-70b":438,"text_davinci_003":89,"gpt4":203,"Yi-34B-Chat":235,"Starling-LM-7B-beta-ExPO":143,"pairrm-Yi-34B-Chat":276,"gpt4_1106_preview":249,"evo-7b":241,"zephyr-7b-beta":238,"guanaco-13b":402,"alpaca-7b":69,"internlm2-chat-20b-ppo":403,"gemma-2b-it":168,"pairrm-zephyr-7b-beta":250,"evo-v2-7b":235,"causallm-14b":234,"SPPO-Mistral7B-PairRM":198,"gpt-3.5-turbo-1106_concise":80,"openbuddy-llama-65b-v8":152,"claude2-alpaca-13b":176,"Starling-LM-7B-alpha-ExPO":220,"openbuddy-falcon-7b-v6":189,"gemma-7b-it":160,"phi-2-sft":164,"gpt4_gamed":81,"llama-2-70b-chat-hf":222,"openbuddy-llama2-70b-v10.1":161,"wizardlm-70b":231,"ultralm-13b-best-of-16":164}}

data/model_win_rates.jsonl

@@ -0,0 +1,53 @@
+{"num_words_mean":257,"num_words_std":145,"win_rate":34.4633736232,"standard_error":1.3146665263,"n_wins":225.0,"n_wins_base":475.0,"n_draws":105.0,"n_total":805.0,"discrete_win_rate":34.4720496894,"length_controlled_winrate":41.8230717152,"lc_standard_error":0.7776876699,"num_tokens_mean":354,"num_tokens_std":206,"model_name":"aligner-2b_claude-3-opus-20240229"}
+{"num_words_mean":253,"num_words_std":153,"win_rate":33.7770952757,"standard_error":1.3776163154,"n_wins":255.0,"n_wins_base":545.0,"n_draws":5.0,"n_total":805.0,"discrete_win_rate":31.9875776398,"length_controlled_winrate":43.905552211,"lc_standard_error":0.8945807936,"num_tokens_mean":346,"num_tokens_std":211,"model_name":"Qwen1.5-110B-Chat"}
+{"num_words_mean":216,"num_words_std":113,"win_rate":29.1052695333,"standard_error":1.3941539442,"n_wins":223.0,"n_wins_base":579.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":27.8881987578,"length_controlled_winrate":40.5095080124,"lc_standard_error":0.8837504763,"num_tokens_mean":292,"num_tokens_std":156,"model_name":"claude-3-opus-20240229"}
+{"num_words_mean":241,"num_words_std":166,"win_rate":21.8557725437,"standard_error":1.2682402187,"n_wins":164.0,"n_wins_base":639.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":20.4968944099,"length_controlled_winrate":28.6143374017,"lc_standard_error":0.9075464438,"num_tokens_mean":327,"num_tokens_std":305,"model_name":"mistral-medium"}
+{"num_words_mean":174,"num_words_std":90,"win_rate":17.1882403567,"standard_error":1.1748282562,"n_wins":131.0,"n_wins_base":673.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":16.3354037267,"length_controlled_winrate":28.1551961416,"lc_standard_error":0.8779084794,"num_tokens_mean":227,"num_tokens_std":114,"model_name":"claude-2"}
+{"num_words_mean":342,"num_words_std":187,"win_rate":35.9464864409,"standard_error":1.4410058098,"n_wins":285.0,"n_wins_base":517.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":35.5900621118,"length_controlled_winrate":34.787447623,"lc_standard_error":0.7594505141,"num_tokens_mean":505,"num_tokens_std":311,"model_name":"FsfairX-Zephyr-Chat-v0.1"}
+{"num_words_mean":79,"num_words_std":78,"win_rate":15.7478281307,"standard_error":1.1194852006,"n_wins":118.0,"n_wins_base":687.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":14.6583850932,"length_controlled_winrate":25.5015577947,"lc_standard_error":0.7760697229,"num_tokens_mean":106,"num_tokens_std":102,"model_name":"Infinity-Instruct-3M-0613-Mistral-7B"}
+{"num_words_mean":243,"num_words_std":144,"win_rate":26.4982833956,"standard_error":1.3042361649,"n_wins":201.0,"n_wins_base":600.0,"n_draws":4.0,"n_total":805.0,"discrete_win_rate":25.2173913043,"length_controlled_winrate":36.571754112,"lc_standard_error":0.9357421321,"num_tokens_mean":342,"num_tokens_std":346,"model_name":"Qwen1.5-72B-Chat"}
+{"num_words_mean":229,"num_words_std":168,"win_rate":22.2101705475,"standard_error":1.2780740057,"n_wins":174.0,"n_wins_base":628.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":21.801242236,"length_controlled_winrate":30.8788102941,"lc_standard_error":0.9518125819,"num_tokens_mean":307,"num_tokens_std":253,"model_name":"Mixtral-8x22B-Instruct-v0.1"}
+{"num_words_mean":235,"num_words_std":143,"win_rate":19.7553327319,"standard_error":1.2063251121,"n_wins":147.0,"n_wins_base":657.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":18.3229813665,"length_controlled_winrate":25.1853410397,"lc_standard_error":0.8999456518,"num_tokens_mean":310,"num_tokens_std":193,"model_name":"dbrx-instruct"}
+{"num_words_mean":297,"num_words_std":161,"win_rate":56.5930456223,"standard_error":1.4464848562,"n_wins":456.0,"n_wins_base":347.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":56.7701863354,"length_controlled_winrate":59.1415240989,"lc_standard_error":0.7580510219,"num_tokens_mean":420,"num_tokens_std":233,"model_name":"Together-MoA-Lite"}
+{"num_words_mean":299,"num_words_std":171,"win_rate":33.1778569588,"standard_error":1.3886514096,"n_wins":266.0,"n_wins_base":537.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":33.1677018634,"length_controlled_winrate":34.4245971745,"lc_standard_error":0.8691832384,"num_tokens_mean":416,"num_tokens_std":238,"model_name":"Meta-Llama-3-70B-Instruct"}
+{"num_words_mean":179,"num_words_std":104,"win_rate":16.1273996216,"standard_error":1.1341036838,"n_wins":120.0,"n_wins_base":682.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":15.0931677019,"length_controlled_winrate":25.6122590254,"lc_standard_error":0.87464248,"num_tokens_mean":233,"num_tokens_std":132,"model_name":"claude-instant-1.2"}
+{"num_words_mean":272,"num_words_std":151,"win_rate":59.8688062333,"standard_error":1.4343056045,"n_wins":490.0,"n_wins_base":314.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":60.9316770186,"length_controlled_winrate":65.3799697685,"lc_standard_error":0.7392392837,"num_tokens_mean":386,"num_tokens_std":217,"model_name":"Together-MoA"}
+{"num_words_mean":218,"num_words_std":148,"win_rate":21.4387759814,"standard_error":1.2485232545,"n_wins":166.0,"n_wins_base":638.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":20.6832298137,"length_controlled_winrate":32.6520799853,"lc_standard_error":0.9044632955,"num_tokens_mean":290,"num_tokens_std":191,"model_name":"mistral-large-2402"}
+{"num_words_mean":336,"num_words_std":195,"win_rate":63.0409907519,"standard_error":1.4253258915,"n_wins":519.0,"n_wins_base":286.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":64.4720496894,"length_controlled_winrate":61.637895572,"lc_standard_error":0.6799412402,"num_tokens_mean":486,"num_tokens_std":287,"model_name":"Storm-7B-best-of-64"}
+{"num_words_mean":221,"num_words_std":129,"win_rate":25.5563252923,"standard_error":1.3419811052,"n_wins":193.0,"n_wins_base":608.0,"n_draws":4.0,"n_total":805.0,"discrete_win_rate":24.2236024845,"length_controlled_winrate":34.8724743624,"lc_standard_error":0.949844689,"num_tokens_mean":297,"num_tokens_std":178,"model_name":"claude-3-sonnet-20240229"}
+{"num_words_mean":176,"num_words_std":97,"win_rate":16.9853436124,"standard_error":1.1687959793,"n_wins":129.0,"n_wins_base":676.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":16.0248447205,"length_controlled_winrate":27.2895044437,"lc_standard_error":0.858614564,"num_tokens_mean":229,"num_tokens_std":123,"model_name":"claude"}
+{"num_words_mean":415,"num_words_std":173,"win_rate":39.354502072,"standard_error":1.4524224246,"n_wins":323.0,"n_wins_base":480.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":40.248447205,"length_controlled_winrate":25.2417704867,"lc_standard_error":0.5909370499,"num_tokens_mean":561,"num_tokens_std":239,"model_name":"Nanbeige2-8B-Chat"}
+{"num_words_mean":341,"num_words_std":273,"win_rate":34.3064238313,"standard_error":1.3914900256,"n_wins":268.0,"n_wins_base":537.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":33.2919254658,"length_controlled_winrate":31.4699427971,"lc_standard_error":0.8138922262,"num_tokens_mean":509,"num_tokens_std":463,"model_name":"REBEL-Llama-3-8B-Instruct"}
+{"num_words_mean":286,"num_words_std":190,"win_rate":51.3275757825,"standard_error":1.470009459,"n_wins":429.0,"n_wins_base":369.0,"n_draws":7.0,"n_total":805.0,"discrete_win_rate":53.7267080745,"length_controlled_winrate":57.4568288333,"lc_standard_error":0.7774399385,"num_tokens_mean":406,"num_tokens_std":273,"model_name":"gpt-4o-2024-05-13"}
+{"num_words_mean":378,"num_words_std":190,"win_rate":64.303601471,"standard_error":1.3348590089,"n_wins":525.0,"n_wins_base":268.0,"n_draws":12.0,"n_total":805.0,"discrete_win_rate":65.9627329193,"length_controlled_winrate":51.5750079797,"lc_standard_error":0.8313707608,"num_tokens_mean":505,"num_tokens_std":261,"model_name":"gpt4_1106_preview_verbose"}
+{"num_words_mean":284,"num_words_std":151,"win_rate":37.0360860499,"standard_error":1.4340261273,"n_wins":288.0,"n_wins_base":514.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":35.9627329193,"length_controlled_winrate":40.5912863493,"lc_standard_error":0.8504106275,"num_tokens_mean":394,"num_tokens_std":214,"model_name":"Nanbeige2-16B-Chat"}
+{"num_words_mean":183,"num_words_std":124,"win_rate":15.7550380876,"standard_error":1.0754642482,"n_wins":117.0,"n_wins_base":684.0,"n_draws":4.0,"n_total":805.0,"discrete_win_rate":14.7826086957,"length_controlled_winrate":30.1833223167,"lc_standard_error":0.7874508454,"num_tokens_mean":245,"num_tokens_std":164,"model_name":"gpt4_0613"}
+{"num_words_mean":313,"num_words_std":180,"win_rate":54.9665397329,"standard_error":1.4286740089,"n_wins":446.0,"n_wins_base":347.0,"n_draws":12.0,"n_total":805.0,"discrete_win_rate":56.149068323,"length_controlled_winrate":56.3562938462,"lc_standard_error":0.7731843456,"num_tokens_mean":417,"num_tokens_std":242,"model_name":"gpt-4-0125-preview"}
+{"num_words_mean":237,"num_words_std":127,"win_rate":23.2373600435,"standard_error":1.2835395056,"n_wins":171.0,"n_wins_base":630.0,"n_draws":4.0,"n_total":805.0,"discrete_win_rate":21.4906832298,"length_controlled_winrate":33.8212668866,"lc_standard_error":0.8842151461,"num_tokens_mean":313,"num_tokens_std":166,"model_name":"gpt4_0613_verbose"}
+{"num_words_mean":391,"num_words_std":159,"win_rate":56.7030097302,"standard_error":1.482841875,"n_wins":456.0,"n_wins_base":347.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":56.7701863354,"length_controlled_winrate":44.4596624034,"lc_standard_error":0.7209678864,"num_tokens_mean":529,"num_tokens_std":216,"model_name":"Nanbeige-Plus-Chat-v0.1"}
+{"num_words_mean":393,"num_words_std":185,"win_rate":34.8601328913,"standard_error":1.3599450437,"n_wins":270.0,"n_wins_base":533.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":33.6645962733,"length_controlled_winrate":29.9743216131,"lc_standard_error":0.7464891533,"num_tokens_mean":526,"num_tokens_std":255,"model_name":"Snorkel-Mistral-PairRM-DPO-best-of-16"}
+{"num_words_mean":228,"num_words_std":94,"win_rate":24.3540710901,"standard_error":1.29358621,"n_wins":191.0,"n_wins_base":613.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":23.7888198758,"length_controlled_winrate":30.2911791666,"lc_standard_error":0.6612722747,"num_tokens_mean":292,"num_tokens_std":120,"model_name":"claude-2.1_verbose"}
+{"num_words_mean":281,"num_words_std":128,"win_rate":29.8963508407,"standard_error":1.3666520485,"n_wins":234.0,"n_wins_base":571.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":29.0683229814,"length_controlled_winrate":31.721885287,"lc_standard_error":0.8150560619,"num_tokens_mean":385,"num_tokens_std":180,"model_name":"merlinite-7B-AOT"}
+{"num_words_mean":215,"num_words_std":160,"win_rate":22.0732589287,"standard_error":1.2466725495,"n_wins":172.0,"n_wins_base":627.0,"n_draws":6.0,"n_total":805.0,"discrete_win_rate":21.7391304348,"length_controlled_winrate":35.3070612164,"lc_standard_error":0.8997916758,"num_tokens_mean":289,"num_tokens_std":212,"model_name":"gpt4_0314"}
+{"num_words_mean":381,"num_words_std":205,"win_rate":33.2273552,"standard_error":1.3779687478,"n_wins":260.0,"n_wins_base":544.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":32.3602484472,"length_controlled_winrate":29.7058089397,"lc_standard_error":0.7122554396,"num_tokens_mean":506,"num_tokens_std":272,"model_name":"Contextual-KTO-Mistral-PairRM"}
+{"num_words_mean":344,"num_words_std":144,"win_rate":35.4431306717,"standard_error":1.3981308966,"n_wins":274.0,"n_wins_base":531.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":34.0372670807,"length_controlled_winrate":31.9003876312,"lc_standard_error":0.7655500294,"num_tokens_mean":459,"num_tokens_std":193,"model_name":"SPPO-Mistral7B-PairRM-ExPO"}
+{"num_words_mean":177,"num_words_std":130,"win_rate":22.9201944405,"standard_error":1.2325177143,"n_wins":172.0,"n_wins_base":622.0,"n_draws":11.0,"n_total":805.0,"discrete_win_rate":22.049689441,"length_controlled_winrate":41.8966015912,"lc_standard_error":0.7406558917,"num_tokens_mean":244,"num_tokens_std":184,"model_name":"gpt4_1106_preview_concise"}
+{"num_words_mean":261,"num_words_std":134,"win_rate":40.6328540086,"standard_error":1.4439449942,"n_wins":325.0,"n_wins_base":479.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":40.4347826087,"length_controlled_winrate":45.807978034,"lc_standard_error":0.8703329817,"num_tokens_mean":370,"num_tokens_std":191,"model_name":"Llama-3-Instruct-8B-SimPO-ExPO"}
+{"num_words_mean":272,"num_words_std":137,"win_rate":63.1549345123,"standard_error":1.4229800988,"n_wins":515.0,"n_wins_base":283.0,"n_draws":7.0,"n_total":805.0,"discrete_win_rate":64.4099378882,"length_controlled_winrate":68.3786625033,"lc_standard_error":0.7309418615,"num_tokens_mean":377,"num_tokens_std":193,"model_name":"openpipe-moa-gpt-4-turbo-v1"}
+{"num_words_mean":225,"num_words_std":140,"win_rate":26.9882543183,"standard_error":1.318903,"n_wins":201.0,"n_wins_base":601.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":25.1552795031,"length_controlled_winrate":31.5065442681,"lc_standard_error":0.7338723477,"num_tokens_mean":312,"num_tokens_std":199,"model_name":"Samba-CoE-v0.2-best-of-16"}
+{"num_words_mean":280,"num_words_std":151,"win_rate":31.773037737,"standard_error":1.2392772646,"n_wins":180.0,"n_wins_base":473.0,"n_draws":152.0,"n_total":805.0,"discrete_win_rate":31.801242236,"length_controlled_winrate":36.7258688784,"lc_standard_error":0.6787999003,"num_tokens_mean":383,"num_tokens_std":210,"model_name":"aligner-2b_qwen1.5-72b-chat"}
+{"num_words_mean":177,"num_words_std":92,"win_rate":15.7335067364,"standard_error":1.1203158654,"n_wins":115.0,"n_wins_base":688.0,"n_draws":2.0,"n_total":805.0,"discrete_win_rate":14.4099378882,"length_controlled_winrate":25.2519438861,"lc_standard_error":0.7515108894,"num_tokens_mean":228,"num_tokens_std":114,"model_name":"claude-2.1"}
+{"num_words_mean":317,"num_words_std":148,"win_rate":39.6728609061,"standard_error":1.4247223562,"n_wins":310.0,"n_wins_base":494.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":38.5714285714,"length_controlled_winrate":38.5628066368,"lc_standard_error":0.8694594533,"num_tokens_mean":443,"num_tokens_std":208,"model_name":"SPPO-Llama-3-Instruct-8B-PairRM"}
+{"num_words_mean":272,"num_words_std":141,"win_rate":40.5297749846,"standard_error":1.4225744647,"n_wins":319.0,"n_wins_base":485.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":39.6894409938,"length_controlled_winrate":44.6804680926,"lc_standard_error":0.8789917177,"num_tokens_mean":385,"num_tokens_std":200,"model_name":"Llama-3-Instruct-8B-SimPO"}
+{"num_words_mean":276,"num_words_std":140,"win_rate":22.9806197059,"standard_error":1.3591734083,"n_wins":184.0,"n_wins_base":620.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":22.9192546584,"length_controlled_winrate":25.7233081711,"lc_standard_error":0.4593179402,"num_tokens_mean":368,"num_tokens_std":203,"model_name":"tulu-2-dpo-70b-ExPO"}
+{"num_words_mean":228,"num_words_std":142,"win_rate":40.5602140968,"standard_error":1.4679655404,"n_wins":312.0,"n_wins_base":493.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":38.7577639752,"length_controlled_winrate":52.3667542714,"lc_standard_error":0.7976856335,"num_tokens_mean":315,"num_tokens_std":200,"model_name":"claude-3-5-sonnet-20240620"}
+{"num_words_mean":300,"num_words_std":135,"win_rate":50.2688690553,"standard_error":1.4728176781,"n_wins":397.0,"n_wins_base":408.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":49.3167701863,"length_controlled_winrate":50.4080792281,"lc_standard_error":0.7188927916,"num_tokens_mean":429,"num_tokens_std":188,"model_name":"Storm-7B"}
+{"num_words_mean":414,"num_words_std":241,"win_rate":30.2200527007,"standard_error":1.3328273013,"n_wins":231.0,"n_wins_base":572.0,"n_draws":1.0,"n_total":804.0,"discrete_win_rate":28.7935323383,"length_controlled_winrate":26.3914464573,"lc_standard_error":0.6739888325,"num_tokens_mean":566,"num_tokens_std":342,"model_name":"Snorkel-Mistral-PairRM-DPO"}
+{"num_words_mean":507,"num_words_std":206,"win_rate":46.1853674689,"standard_error":1.4638315246,"n_wins":375.0,"n_wins_base":430.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":46.5838509317,"length_controlled_winrate":27.2257594808,"lc_standard_error":0.5877331102,"num_tokens_mean":693,"num_tokens_std":283,"model_name":"internlm2-chat-20b-ExPO"}
+{"num_words_mean":210,"num_words_std":138,"win_rate":21.8473786693,"standard_error":1.2171089783,"n_wins":159.0,"n_wins_base":645.0,"n_draws":1.0,"n_total":805.0,"discrete_win_rate":19.8136645963,"length_controlled_winrate":27.6242673501,"lc_standard_error":0.6875926799,"num_tokens_mean":292,"num_tokens_std":196,"model_name":"Samba-CoE-v0.2"}
+{"num_words_mean":215,"num_words_std":156,"win_rate":23.5767893148,"standard_error":1.2757042012,"n_wins":179.0,"n_wins_base":618.0,"n_draws":8.0,"n_total":805.0,"discrete_win_rate":22.7329192547,"length_controlled_winrate":38.1280897444,"lc_standard_error":0.9069675584,"num_tokens_mean":287,"num_tokens_std":203,"model_name":"gpt4"}
+{"num_words_mean":339,"num_words_std":178,"win_rate":29.6599467188,"standard_error":1.3225712598,"n_wins":219.0,"n_wins_base":582.0,"n_draws":4.0,"n_total":805.0,"discrete_win_rate":27.4534161491,"length_controlled_winrate":27.1905478776,"lc_standard_error":0.7470363322,"num_tokens_mean":447,"num_tokens_std":235,"model_name":"Yi-34B-Chat"}
+{"num_words_mean":336,"num_words_std":106,"win_rate":29.6008518479,"standard_error":1.3252049543,"n_wins":225.0,"n_wins_base":580.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":27.950310559,"length_controlled_winrate":26.4869564984,"lc_standard_error":0.7549415682,"num_tokens_mean":447,"num_tokens_std":143,"model_name":"Starling-LM-7B-beta-ExPO"}
+{"num_words_mean":349,"num_words_std":189,"win_rate":31.2412829468,"standard_error":1.3482437399,"n_wins":239.0,"n_wins_base":563.0,"n_draws":3.0,"n_total":805.0,"discrete_win_rate":29.8757763975,"length_controlled_winrate":28.8148408668,"lc_standard_error":0.8310750322,"num_tokens_mean":464,"num_tokens_std":276,"model_name":"pairrm-Yi-34B-Chat"}
+{"num_words_mean":323,"num_words_std":181,"win_rate":50.0,"standard_error":0.0,"n_wins":0.0,"n_wins_base":0.0,"n_draws":805.0,"n_total":805.0,"discrete_win_rate":50.0,"length_controlled_winrate":50.0,"lc_standard_error":0.0,"num_tokens_mean":431,"num_tokens_std":249,"model_name":"gpt4_1106_preview"}
+{"num_words_mean":322,"num_words_std":148,"win_rate":32.2453123638,"standard_error":1.390800011,"n_wins":249.0,"n_wins_base":556.0,"n_draws":0.0,"n_total":805.0,"discrete_win_rate":30.9316770186,"length_controlled_winrate":30.4941379652,"lc_standard_error":0.8458266977,"num_tokens_mean":431,"num_tokens_std":198,"model_name":"SPPO-Mistral7B-PairRM"}

prep_data.py

@@ -20,7 +20,9 @@ for model_name in os.listdir(TOP_LEVEL_DIRECTORY):
     if os.path.isdir(model_dir):
         model_output_file = os.path.join(model_dir, "model_outputs.json")
         if os.path.exists(model_output_file):
-            model_dataframes_outputs[model_name] = pd.read_json(model_output_file)
+            df = pd.read_json(model_output_file)
+            df["model_name"] = model_name
+            model_dataframes_outputs[model_name] = df
 
 
 def get_num_words(text):
@@ -42,6 +44,7 @@ model_name_to_num_words = {}
 model_name_to_num_tokens = {}
 for model_name, model_dataframe in model_dataframes_outputs.items():
     print(f"model_name_to_num_words for {model_name}")
+    model_dataframe["model_name"] = model_name
     model_dataframe["output_num_words"] = model_dataframe["output"].apply(get_num_words)
     model_dataframe["output_num_tokens"] = model_dataframe["output"].apply(
         get_num_tokens
@@ -88,5 +91,8 @@ df = df.rename(
         "std": "num_tokens_std",
     }
 )
+df["model_name"] = df.index
 
-df.to_json("data/model_win_rates.json")
+df = df[df["length_controlled_winrate"] > 25]
+
+df.to_json("data/model_win_rates.jsonl", orient="records", lines=True)

prep_data_annotations.py

@@ -0,0 +1,62 @@
+import os
+import json
+import tiktoken
+from alpaca_eval import utils, metrics, annotators, constants, analyze, plotting, main
+from alpaca_eval.metrics.glm_winrate import get_length_controlled_winrate
+import os
+import pandas as pd
+import json
+
+
+# Define the path to the top-level directory
+TOP_LEVEL_DIRECTORY = "submodules/alpaca_eval/results"
+
+df = pd.read_json("data/model_win_rates.jsonl", lines=True, orient="records")
+relevant_models = df["model_name"].unique().tolist()
+
+# Initialize an empty dictionary to hold the model name to dataframe mapping
+model_dataframes_outputs = {}
+
+# Iterate through each subdirectory in the top-level directory
+df_response_judging = pd.DataFrame()
+for model_name in os.listdir(TOP_LEVEL_DIRECTORY):
+    if model_name not in relevant_models:
+        continue
+    model_dir = os.path.join(TOP_LEVEL_DIRECTORY, model_name)
+    if os.path.isdir(model_dir):
+        model_output_file = os.path.join(
+            model_dir, "weighted_alpaca_eval_gpt4_turbo/annotations.json"
+        )
+        if os.path.exists(model_output_file):
+            df_response_judging = pd.concat(
+                [df_response_judging, pd.read_json(model_output_file)]
+            )
+
+df_responses = pd.DataFrame()
+for model_name in os.listdir(TOP_LEVEL_DIRECTORY):
+    if model_name not in relevant_models:
+        continue
+    model_dir = os.path.join(TOP_LEVEL_DIRECTORY, model_name)
+    if os.path.isdir(model_dir):
+        model_output_file = os.path.join(model_dir, "model_outputs.json")
+        if os.path.exists(model_output_file):
+            df_responses = pd.concat([df_responses, pd.read_json(model_output_file)])
+
+df_responses = df_responses.drop("all_generated_texts", axis=1)
+df_responses = df_responses.drop("Unnamed: 0.1", axis=1)
+df_responses = df_responses.drop("index", axis=1)
+df_responses = df_responses.drop("Unnamed: 0", axis=1)
+df_responses = df_responses.drop("scores", axis=1)
+df_responses = df_responses.drop("all_results_idx_best", axis=1)
+df_responses = df_responses.drop("original_output", axis=1)
+df_responses = df_responses.drop("new_prompt", axis=1)
+
+breakpoint()
+
+# Whitelist.
+
+
+df_response_judging.to_json(
+    "data/df_response_judging.jsonl", lines=True, orient="records"
+)
+df_responses.to_json("data/df_responses.jsonl", lines=True, orient="records")