app.py

import streamlit as st
import pandas as pd

st.title("‎‎‎  ‎‎ ‎ ‎ ‎ ‎ ‎ ‎🏋️‍♂️ benchbench-Leaderboard 🏋️‍♂️")

import pandas as pd
from bat import Tester, Config, Benchmark, Reporter
from bat.utils import get_holistic_benchmark


cfg = Config(
    exp_to_run="example",
    n_models_taken_list=[0],
    model_select_strategy_list=["random"],
    n_exps=10,
    # reference_data_path="data/combined_holistic.csv",
)


newbench_name = "livebench"
new_bench_agg_name = f"{newbench_name}_mwr"

tester = Tester(cfg=cfg)

# models_for_benchmark_scoring = tester.fetch_reference_models_names(
#     reference_benchmark=get_holistic_benchmark(), n_models=20
# )

newbench = Benchmark(
    pd.read_csv(f"assets/{newbench_name}.csv"),
    data_source=newbench_name,
)

# newbench.add_aggragete(new_col_name=new_bench_agg_name)
# newbench_agreements = tester.all_vs_all_agreement_testing(newbench)

reporter = Reporter()
# reporter.draw_agreements(
#     newbench_agreements, ref_sources=[newbench_name], scenario_sources=[newbench_name]
# )

holistic = get_holistic_benchmark()
holistic.add_aggragete(new_col_name="aggregate", agg_source_name="holistic")

allbench = newbench.extend(holistic)
allbench.clear_repeated_scenarios(source_to_keep=newbench_name)


@st.cache_data
def run_load():
    return tester.all_vs_all_agreement_testing(allbench)


all_agreements = run_load()

observed_scenario = "arena_elo"  # "livebench_lb"
blacklist_sources = []  # "livebench"

z_score = reporter.get_z_score(all_agreements, observed_scenario, blacklist_sources)

st.write(f"zscore of {observed_scenario}: {z_score}")

# df = pd.read_csv("BAT_w_arena_10_random.csv")
# df = (
#     (
#         df.rename(
#             columns={
#                 "z_score": "Z_Score",
#                 "benchmark": "Benchmark",
#             }
#         ).drop(
#             columns=[
#                 "Unnamed: 0",
#                 "z_test_pass",
#             ]
#         )
#     )
#     .sort_values("Z_Score", ascending=False)
#     .query(
#         'Benchmark!="Aggregate" and Benchmark!="MAGI" and Benchmark!="Alpaca(v2, len adj)" and Benchmark!="GPT4All"'
#     )
# )


# df.replace(
#     {
#         "Arena Elo": "LMSys Arena",
#         "Hugging-6": "HF OpenLLM",
#         "Alpaca(v2)": "Alpaca v2",
#         "Alpaca(v1)": "Alpaca v1",
#         "EQ-Bench(v2)": "EQ-Bench v2",
#     },
#     inplace=True,
# )

# col1, col2, col3 = st.columns(3)

# with col1:
#     st.header("‎ ‎ ‎ ‎ ‎ ‎ ‎ ‎ Agree")
#     st.dataframe(df.query("Z_Score>=0"), hide_index=True)

# with col2:
#     st.header("‎ ‎‎  ‎ Disagree")
#     st.dataframe(df.query("Z_Score<0").sort_values("Z_Score"), hide_index=True)

# with col3:
#     st.header("‎ ‎‎  ‎ Configs")
#     # st.selectbox(label="Reference Benchmarks", options=["LMSys Arena"])
#     options = st.multiselect(
#         "Reference Benchmarks",
#         ["LMSys Arena", "Open Compass", "Yellow", "Red", "Blue"],
#         ["LMSys Arena", "Open Compass"],
#     )
#     st.selectbox(label="# models compared", options=[20])
#     st.selectbox(label="Model Select Strategy", options=["Random"])
#     st.write("‎‎‎‎‎‎‎")
#     st.button("Upload a new benchmark")