update ko_bench

app.py

@@ -1,7 +1,6 @@
 import gradio as gr
 import pandas as pd
 import numpy as np
-import plotly.express as px
 import random
 import plotly.graph_objects as go
 
@@ -62,7 +61,7 @@ def get_license(model): # 대소문자 무시하고 모델을 매칭하기 위
 
 # dataframe_full
 df_full_rs = df_rs.copy()
-df_full_rs.rename(columns={'score': 'Ko-Bench'}, inplace=True)
+df_full_rs.rename(columns={'score': 'KO-Bench'}, inplace=True)
 df_full_rs = df_full_rs.drop(columns=['Coding', 'Extraction', 'Humanities', 'Math', 'Reasoning', 'Roleplay', 'STEM', 'Writing'])
 
 df_full_rs = df_full_rs.drop(columns=['turn']) # 모델별 turn1,2 score 합병
@@ -70,16 +69,16 @@ df_full_rs = df_full_rs.groupby(['model', 'judge_model']).agg({col: custom_mean
 df_full_rs = df_full_rs.round(2)
 df_full_rs.replace("", np.nan, inplace=True)
 
-df_full_rs['Ko-Bench/openai'] = '' # Ko-Bench/openai, Ko-Bench/keval 열 추가
-df_full_rs['Ko-Bench/keval'] = ''
+df_full_rs['KO-Bench/openai'] = '' # KO-Bench/openai, KO-Bench/keval 열 추가
+df_full_rs['KO-Bench/keval'] = ''
 for idx, j_model in df_full_rs['judge_model'].items():
     if j_model == 'keval':
-        df_full_rs.at[idx, 'Ko-Bench/keval'] = df_full_rs.at[idx, 'Ko-Bench']
+        df_full_rs.at[idx, 'KO-Bench/keval'] = df_full_rs.at[idx, 'KO-Bench']
     else :
-        df_full_rs.at[idx, 'Ko-Bench/openai'] = df_full_rs.at[idx, 'Ko-Bench']
+        df_full_rs.at[idx, 'KO-Bench/openai'] = df_full_rs.at[idx, 'KO-Bench']
 df_full_rs = df_full_rs.drop(columns=['judge_model'])
 
-df_full_rs = df_full_rs.groupby(['model']).agg({col: custom_mean for col in df_full_rs.columns if col not in ['model']}).reset_index() # Ko-Bench/openai, Ko-Bench/keval 행 합병
+df_full_rs = df_full_rs.groupby(['model']).agg({col: custom_mean for col in df_full_rs.columns if col not in ['model']}).reset_index() # KO-Bench/openai, KO-Bench/keval 행 합병
 df_full_rs = df_full_rs.round(2)
 df_full_rs.replace("", np.nan, inplace=True)
 
@@ -93,9 +92,9 @@ df_full_rs['Organization'] = df_full_rs['model'].apply(get_organization)
 df_full_rs['License'] = '' # License 열 추가
 df_full_rs['License'] = df_full_rs['model'].apply(get_license)
 
-df_full_rs = df_full_rs.sort_values(by='Ko-Bench', ascending=False)
+df_full_rs = df_full_rs.sort_values(by='KO-Bench', ascending=False)
 df_full_rs.insert(0, 'rank', range(1, len(df_full_rs) + 1))
-df_full_rs = df_full_rs.drop(columns=['Ko-Bench'])
+df_full_rs = df_full_rs.drop(columns=['KO-Bench'])
 
 plot_models = df_full_rs['model'].unique() # model detail view를 위한 models 리스트
 
@@ -135,7 +134,8 @@ df_keval.insert(0, 'rank', range(1, len(df_keval) + 1))
 # model detail view
 plot_models_list = plot_models.tolist()
 CATEGORIES = ["Writing", "Roleplay", "Reasoning", "Math", "Coding", "Extraction", "STEM", "Humanities"]
-category_labels = ['Selected model turn1', 'Selected model turn2', 'Top1 turn1', 'Top1 turn2']
+colors_openai = ['#ff0000', '#ff1493', '#115e02', '#21ad05']
+colors_keval = ['#ff0000', '#ff1493', '#0000ff', '#0592eb']
 random.seed(42)
 
 def search_dataframe(query): # df 검색 함수 정의
@@ -144,32 +144,36 @@ def search_dataframe(query): # df 검색 함수 정의
     filtered_df = df[df.apply(lambda row: any(row.astype(str) == query), axis=1)]
     return filtered_df
 
-def radar_chart(categories, Selected_model_turn1, Selected_model_turn2, Top1_turn1, Top1_turn2): # plot 그리는 함수
+def radar_chart(categories, Top1_turn1, Top1_turn2, Selected_model_turn1, Selected_model_turn2, category_labels, str): # plot 그리는 함수
     #categories = categories.split(',')
-
-    Selected_model_turn1 = [item for sublist in Selected_model_turn1 for item in sublist]
-    Selected_model_turn2 = [item for sublist in Selected_model_turn2 for item in sublist]
     Top1_turn1 = [item for sublist in Top1_turn1 for item in sublist]
     Top1_turn2 = [item for sublist in Top1_turn2 for item in sublist]
+    Selected_model_turn1 = [item for sublist in Selected_model_turn1 for item in sublist]
+    Selected_model_turn2 = [item for sublist in Selected_model_turn2 for item in sublist]
 
     values_lists = [
-        list(map(float, Selected_model_turn1)),
-        list(map(float, Selected_model_turn2)),
         list(map(float, Top1_turn1)),
-        list(map(float, Top1_turn2))
+        list(map(float, Top1_turn2)),
+        list(map(float, Selected_model_turn1)),
+        list(map(float, Selected_model_turn2))
     ]
 
+    if str == "openai": colors = colors_openai
+    else: colors = colors_keval
+    if str == "openai": title_text = "< Openai >"
+    else: title_text = "< Keval >"
+
     fig = go.Figure()
 
     for i, values in enumerate(values_lists):
         if len(categories) != len(values):
             return f"Error in dataset {i+1}: Number of categories and values must be the same."
-
         fig.add_trace(go.Scatterpolar(
             r=values + [values[0]],  # Closing the loop of the radar chart
             theta=categories + [categories[0]],  # Closing the loop of the radar chart
             mode='lines',
-            name=category_labels[i] # Label for the dataset
+            name=category_labels[i], # Label for the dataset
+            line = dict(color= colors[i])
         ))
 
     fig.update_layout(
@@ -185,63 +189,82 @@ def radar_chart(categories, Selected_model_turn1, Selected_model_turn2, Top1_tur
             )
         ),
         showlegend=True,
-        width=555,  # 적절한 너비 설정
-        height=550,  # 적절한 높이 설정
+        #width=650,  # 적절한 너비 설정
+        #height=650,  # 적절한 높이 설정
         margin=dict(l=1000, r=20, t=20, b=20),
-        autosize = False,
+        #autosize = False,
         paper_bgcolor='white',
-        plot_bgcolor='lightgrey'
+        plot_bgcolor='lightgrey',
+        title=dict(
+            text=title_text,  # 제목을 원하는 텍스트로 변경
+            x=0.5,  # 제목의 x 위치 (0=왼쪽, 0.5=중앙, 1=오른쪽)
+            xanchor='center',  # 제목의 x 위치 기준 (center, left, right)
+            y=0.95,  # 제목의 y 위치 (0=하단, 1=상단)
+            yanchor='top'  # 제목의 y 위치 기준 (top, middle, bottom)
+        )
     )
     return fig
 
 def search_openai_plot(dropdown_model): # openai plot 함수 정의
-    condition1 = (df['judge_model'] != 'keval') & (df['turn'] == 1) & (df['model'] == dropdown_model)
-    openai_turn1 = df.loc[condition1, 'Coding':'Writing'].values.tolist()
+    condition1 = (df['judge_model'] != 'keval') & (df['turn'] == 1) & (df['model'] == df_openai.iat[0, df_openai.columns.get_loc('model')])
+    top1_openai_turn1 = df.loc[condition1, 'Coding':'Writing'].values.tolist()
+
+    condition2 = (df['judge_model'] != 'keval') & (df['turn'] == 2) & (df['model'] == df_openai.iat[0, df_openai.columns.get_loc('model')])
+    top1_openai_turn2 = df.loc[condition2, 'Coding':'Writing'].values.tolist()
 
-    condition2 = (df['judge_model'] != 'keval') & (df['turn'] == 2) & (df['model'] == dropdown_model)
-    openai_turn2 = df.loc[condition2, 'Coding':'Writing'].values.tolist()
+    condition3 = (df['judge_model'] != 'keval') & (df['turn'] == 1) & (df['model'] == dropdown_model)
+    openai_turn1 = df.loc[condition3, 'Coding':'Writing'].values.tolist()
 
-    condition3 = (df['judge_model'] != 'keval') & (df['turn'] == 1) & (df['model'] == df_openai.loc[0,'model'])
-    top1_openai_turn1 = df.loc[condition3, 'Coding':'Writing'].values.tolist()
+    condition4 = (df['judge_model'] != 'keval') & (df['turn'] == 2) & (df['model'] == dropdown_model)
+    openai_turn2 = df.loc[condition4, 'Coding':'Writing'].values.tolist()
 
-    condition4 = (df['judge_model'] != 'keval') & (df['turn'] == 2) & (df['model'] == df_openai.loc[0,'model'])
-    top1_openai_turn2 = df.loc[condition4, 'Coding':'Writing'].values.tolist()
+    category_labels = []
+    category_labels.append(df_openai.iat[0, df_openai.columns.get_loc('model')] + "  /Turn 1")
+    category_labels.append(df_openai.iat[0, df_openai.columns.get_loc('model')] + "  /Turn 2")
+    category_labels.append(dropdown_model + "  /Turn 1")
+    category_labels.append(dropdown_model + "  /Turn 2")
 
-    fig = radar_chart(CATEGORIES, openai_turn1, openai_turn2, top1_openai_turn1, top1_openai_turn2)
+    fig = radar_chart(CATEGORIES, top1_openai_turn1, top1_openai_turn2, openai_turn1, openai_turn2, category_labels,"openai")
     return fig
 
 def search_keval_plot(dropdown_model): # keval plot 함수 정의
-    condition1 = (df['judge_model'] == 'keval') & (df['turn'] == 1) & (df['model'] == dropdown_model)
-    keval_turn1 = df.loc[condition1, 'Coding':'Writing'].values.tolist()
+    condition1 = (df['judge_model'] == 'keval') & (df['turn'] == 1) & (df['model'] == df_keval.iat[0, df_keval.columns.get_loc('model')])
+    top1_keval_turn1 = df.loc[condition1, 'Coding':'Writing'].values.tolist()
 
-    condition2 = (df['judge_model'] == 'keval') & (df['turn'] == 2) & (df['model'] == dropdown_model)
-    keval_turn2 = df.loc[condition2, 'Coding':'Writing'].values.tolist()
+    condition2 = (df['judge_model'] == 'keval') & (df['turn'] == 2) & (df['model'] == df_keval.iat[0, df_keval.columns.get_loc('model')])
+    top1_keval_turn2 = df.loc[condition2, 'Coding':'Writing'].values.tolist()
 
-    condition3 = (df['judge_model'] == 'keval') & (df['turn'] == 1) & (df['model'] == df_keval.loc[0,'model'])
-    top1_keval_turn1 = df.loc[condition3, 'Coding':'Writing'].values.tolist()
+    condition3 = (df['judge_model'] == 'keval') & (df['turn'] == 1) & (df['model'] == dropdown_model)
+    keval_turn1 = df.loc[condition3, 'Coding':'Writing'].values.tolist()
 
-    condition4 = (df['judge_model'] == 'keval') & (df['turn'] == 2) & (df['model'] == df_keval.loc[0,'model'])
-    top1_keval_turn2 = df.loc[condition4, 'Coding':'Writing'].values.tolist()
+    condition4 = (df['judge_model'] == 'keval') & (df['turn'] == 2) & (df['model'] == dropdown_model)
+    keval_turn2 = df.loc[condition4, 'Coding':'Writing'].values.tolist()
 
-    fig = radar_chart(CATEGORIES, keval_turn1, keval_turn2, top1_keval_turn1, top1_keval_turn2)
+    category_labels = []
+    category_labels.append(df_keval.iat[0, df_keval.columns.get_loc('model')] + " /Turn 1")
+    category_labels.append(df_keval.iat[0, df_keval.columns.get_loc('model')] + "  /Turn 2")
+    category_labels.append(dropdown_model + "  /Turn 1")
+    category_labels.append(dropdown_model + "  /Turn 2")
+
+    fig = radar_chart(CATEGORIES, top1_keval_turn1, top1_keval_turn2, keval_turn1, keval_turn2, category_labels, "keval")
     return fig
 
 
 #gradio
 with gr.Blocks() as demo:
     gr.Markdown("")
-    gr.Markdown("# 🏆 Ko-Bench Leaderboard")
+    gr.Markdown("# 🏆 KO-Bench Leaderboard")
+    gr.Markdown("")
     gr.Markdown("")
-    gr.Markdown("#### The Ko-Bench is a leaderboard for evaluating the multi-level conversation ability and instruction-following ability of Korean Large Language Models (LLMs).")
+    gr.Markdown("#### The Ko-bench is a leaderboard for evaluating the multi-level conversation ability and instruction-following ability of Korean Large Language Models (LLMs).")
     gr.Markdown("- MT-Bench: a set of challenging multi-turn questions. We use GPT-4 to grade the model responses.")
-    gr.Markdown("- Ko-Bench/openai: a set of challenging multi-turn questions in Korean. We use GPT-4o to grade the model responses.")
-    gr.Markdown("- Ko-Bench/keval: a set of challenging multi-turn questions in Korean. We use the keval model as an evaluation model.")
+    gr.Markdown("- KO-Bench/openai: a set of challenging multi-turn questions in Korean. We use GPT-4o to grade the model responses.")
+    gr.Markdown("- KO-Bench/keval: a set of challenging multi-turn questions in Korean. We use the keval model as an evaluation model.")
+    gr.Markdown("")
     gr.Markdown("")
-    gr.Markdown("github : https://github.com/davidkim205/Ko-Bench")
-    gr.Markdown("keval : https://huggingface.co/collections/davidkim205/k-eval-6660063dd66e21cbdcc4fbf1")    
     gr.Markdown("")
 
-    with gr.TabItem("Ko-Bench"):
+    with gr.TabItem("KO-Bench"):
         gr.Dataframe(value=df_full_rs)
     with gr.TabItem("Openai Judgment"):
         gr.Dataframe(value=df_openai)
@@ -257,7 +280,6 @@ with gr.Blocks() as demo:
             with gr.Row():
                 plot_openai = gr.Plot(label="Openai Plot")
                 dropdown.change(fn=search_openai_plot, inputs=dropdown, outputs=plot_openai)
-            #with gr.Row():
                 plot_keval = gr.Plot(label="Keval Plot")
                 dropdown.change(fn=search_keval_plot, inputs=dropdown, outputs=plot_keval)
 

ko_bench.csv

@@ -5,6 +5,7 @@ gpt-4o,1,GPT-4o-mini-2024-07-18,8.8,7.3,9.2,9.4,10.0,6.9,8.7,9.6,9.1
 gpt-4o,1,Mistral-Large-Instruct-2407,8.5,6.8,8.9,8.7,9.6,6.6,8.5,9.2,9.5
 gpt-4o,1,Qwen2-72B-Instruct,8.3,5.1,9.7,8.9,7.5,7.9,8.8,9.2,9.3
 gpt-4o,1,gemma-2-27b-it,8.3,6.8,9.4,9.5,7.9,5.4,9.0,9.0,9.2
+gpt-4o,1,gemini-1.5-pro,8.2,5.5,9.7,8.7,7.5,6.5,9.1,9.4,9.2
 gpt-4o,1,ko-gemma-2-9b-it,7.8,6.6,9.0,8.4,6.7,6.2,8.1,8.9,8.7
 gpt-4o,1,gemma-2-9b-it,7.7,6.2,9.3,8.8,5.4,5.4,8.8,8.8,8.7
 gpt-4o,1,WizardLM-2-8x22B,7.4,6.8,6.8,7.8,8.7,4.8,7.2,8.4,8.7
@@ -24,6 +25,7 @@ gpt-4o,2,gpt-4-0125-preview,8.0,7.2,8.5,8.9,6.8,7.3,8.7,8.1,8.6
 gpt-4o,2,GPT-4o-mini-2024-07-18,7.6,6.2,7.6,9.1,7.8,4.6,8.2,9.0,8.3
 gpt-4o,2,Mistral-Large-Instruct-2407,7.2,6.5,8.8,7.5,7.9,4.7,7.3,7.2,7.6
 gpt-4o,2,gemma-2-27b-it,7.0,6.4,7.6,9.0,5.4,5.1,7.9,7.4,7.4
+gpt-4o,2,gemini-1.5-pro,7.0,6.3,7.7,8.3,6.1,5.0,8.5,7.8,6.5
 gpt-4o,2,Qwen2-72B-Instruct,6.9,5.5,8.4,8.7,5.3,4.4,7.9,7.4,7.6
 gpt-4o,2,ko-gemma-2-9b-it,6.4,5.7,6.9,8.5,5.6,4.3,7.3,6.6,6.5
 gpt-4o,2,WizardLM-2-8x22B,6.4,6.0,8.2,7.2,6.1,4.1,7.0,6.8,5.5
@@ -43,6 +45,7 @@ keval,1,GPT-4o-2024-05-13,9.1,7.8,9.5,9.6,9.9,8.8,8.7,9.3,9.2
 keval,1,gpt-4-0125-preview,8.8,7.7,9.6,9.2,9.8,7.5,8.2,9.5,9.2
 keval,1,GPT-4o-mini-2024-07-18,8.7,7.8,8.2,9.3,10.0,6.9,8.8,9.7,9.2
 keval,1,Mistral-Large-Instruct-2407,8.2,6.3,7.9,8.9,9.6,6.4,8.2,9.5,9.2
+keval,1,gemini-1.5-pro,8.2,5.7,9.8,8.8,7.4,6.2,9.1,9.7,9.0
 keval,1,gemma-2-27b-it,8.1,5.9,9.3,9.4,7.4,5.7,8.9,9.0,9.0
 keval,1,Qwen2-72B-Instruct,8.0,5.0,9.2,8.8,8.6,6.9,7.7,9.1,9.0
 keval,1,ko-gemma-2-9b-it,7.8,5.9,9.4,8.5,6.0,6.3,8.2,9.0,8.9
@@ -66,6 +69,7 @@ keval,2,Mistral-Large-Instruct-2407,7.0,5.4,7.3,8.5,7.3,5.2,7.9,7.8,6.9
 keval,2,Qwen2-72B-Instruct,7.0,6.2,7.5,8.7,5.5,5.3,7.5,6.9,8.1
 keval,2,gemma-2-27b-it,6.9,6.6,7.0,8.9,5.5,5.0,7.6,6.9,7.3
 keval,2,WizardLM-2-8x22B,6.6,5.6,7.6,7.9,6.3,4.9,6.9,7.4,6.3
+keval,2,gemini-1.5-pro,6.5,5.2,6.9,8.4,6.0,4.8,8.1,7.3,5.4
 keval,2,ko-gemma-2-9b-it,6.4,5.1,6.6,8.9,6.0,4.0,7.2,6.8,6.7
 keval,2,gemma-2-9b-it,6.3,5.2,7.7,8.7,4.6,4.0,7.8,6.8,5.4
 keval,2,EXAONE-3.0-7.8B-Instruct,6.2,5.9,7.0,6.4,6.7,4.3,7.6,4.2,7.8