Spaces:
Runtime error
Runtime error
File size: 9,007 Bytes
4c59875 6964a16 4c59875 6964a16 4c59875 a888db4 6964a16 a888db4 6964a16 4c59875 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
import pandas as pd
import gradio as gr
import plotly.express as px
import plotly.graph_objects as go
explanation_data = {
"Accuracy Scores (rename for clarity)": [
"b4bqa",
"b4b",
"medmcqa_g2b",
"medmcqa_orig_filtered",
"medmcqa_diff",
"medqa_4options_g2b",
"medqa_4options_orig_filtered",
"medqa_diff"
],
"Description": [
"Model accuracy on the B4BQA task.",
"Average model accuracy on tasks where generic drug names are substituted with brand names.",
"Model accuracy on MedMCQA task where generic drug names are substituted with brand names.",
"Model accuracy on MedMCQA task with original data. (Only includes questions that overlap with the g2b dataset)",
"Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance.",
"Model accuracy on MedQA (4 options) task where generic drug names are substituted with brand names.",
"Model accuracy on MedQA (4 options) task with original data. (Only includes questions that overlap with the g2b dataset)",
"Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance."
]
}
explanation_df = pd.DataFrame(explanation_data)
df = pd.read_csv("data/csv/models_data.csv")
filter_mapping = {
"all": "all",
"🟢 Pre-trained": "🟢",
"🟩 Continuously pre-trained": "🟩",
"🔶 Fine-tuned on domain-specific data": "🔶",
"💬 Chat-models (RLHF, DPO, IFT, ...)": "💬"
}
def filter_items(df, query):
if query == "all":
return df
filter_value = filter_mapping[query]
return df[df["T"].str.contains(filter_value, na=False)]
def create_scatter_plot(df, x_col, y_col, title, x_title, y_title):
fig = px.scatter(df, x=x_col, y=y_col, color='Model', title=title)
fig.add_trace(
go.Scatter(
x=[0, 100],
y=[0, 100],
mode="lines",
name="y=x line",
line=dict(color='black', dash='dash')
)
)
fig.update_layout(
xaxis_title=x_title,
yaxis_title=y_title,
xaxis=dict(range=[0, 100]),
yaxis=dict(range=[0, 100]),
legend_title_text='Model'
)
fig.update_traces(marker=dict(size=10), selector=dict(mode='markers'))
return fig
with gr.Blocks(css="custom.css") as demo:
with gr.Column():
gr.Markdown(
"""<div style="text-align: center;"><h1> <span style='color: #FFC43A;'>🐰 RABBITS:</span> <span style='color: #FFC43A;'>R</span>obust <span style='color: #FFC43A;'>A</span>ssessment of <span style='color: #FFC43A;'>B</span>iomedical <span style='color: #FFC43A;'>B</span>enchmarks <span style='color: #FFC43A;'>I</span>nvolving drug
<span style='color: #FFC43A;'>T</span>erm <span style='color: #FFC43A;'>S</span>ubstitutions for Language Models <span style='color: #FFC43A;'></span></h1></div>"""
)
with gr.Row():
gr.Markdown(""" """)
with gr.Row():
gr.Markdown(
"""<div style="text-align: center;">
<p class='markdown-text'>Robust language models are crucial in the medical domain and the RABBITS project tests the robustness of LLMs by evaluating their handling of synonyms, specifically brand and generic drug names. We assessed 16 open-source language models from Hugging Face using systematic synonym substitution on MedQA and MedMCQA tasks. Our results show a consistent decline in performance across all model sizes, highlighting challenges in synonym comprehension. Additionally, we discovered significant dataset contamination by identifying overlaps between MedQA, MedMCQA test sets, and the Dolma 1.6 dataset using an 8-gram analysis. This highlights the need to improve model robustness and address contamination in open-source datasets.</p>
</div>"""
)
with gr.Row():
gr.Markdown(""" """)
with gr.Row():
gr.Image(value="workflow-1-2.svg", width=200, height=450)
gr.Image(value="workflow-3-4.svg", width=200, height=450)
with gr.Row():
gr.Markdown(""" """)
with gr.Row():
gr.Dataframe(
value=explanation_df,
headers="keys",
datatype=["str", "str"],
interactive=False,
label="Explanation of Scores"
)
with gr.Row():
gr.Markdown(""" """)
with gr.Tabs(elem_classes="tab-buttons"):
with gr.TabItem("🔍 Evaluation table"):
with gr.Column():
with gr.Accordion("➡️ Filter by Column", open=False):
shown_columns = gr.CheckboxGroup(
choices=df.columns.tolist(),
value=df.columns.tolist(),
label="Select Columns",
interactive=True,
)
with gr.Row():
search_bar = gr.Textbox(
placeholder="🔍 Search for your model and press ENTER...",
show_label=False,
elem_id="search-bar"
)
filter_columns = gr.Radio(
label="⏚ Filter model types",
choices=[
"all",
"🟢 Pre-trained",
"🟩 Continuously pre-trained",
"🔶 Fine-tuned on domain-specific data",
"💬 Chat-models (RLHF, DPO, IFT, ...)"
],
value="all",
elem_id="filter-columns",
)
leaderboard_df = gr.Dataframe(
value=df,
headers="keys",
datatype=["html" if col == "Model" else "str" for col in df.columns],
interactive=False,
elem_id="leaderboard-table"
)
def update_leaderboard(search_query):
filtered_df = df[df["Model"].str.contains(search_query, case=False)]
return filtered_df
search_bar.submit(
update_leaderboard,
inputs=search_bar,
outputs=leaderboard_df
)
def filter_update(query):
filtered_df = filter_items(df, query)
return filtered_df
filter_columns.change(
filter_update,
inputs=filter_columns,
outputs=leaderboard_df
)
shown_columns.change(
lambda cols: df[cols],
inputs=shown_columns,
outputs=leaderboard_df
)
with gr.TabItem("📊 Evaluation Plots"):
with gr.Column():
with gr.Row():
scatter1 = gr.Plot(
value=create_scatter_plot(df, "medmcqa_orig_filtered", "medmcqa_g2b",
"MedMCQA: Orig vs G2B", "medmcqa_orig_filtered", "medmcqa_g2b"),
elem_id="scatter1"
)
scatter2 = gr.Plot(
value=create_scatter_plot(df, "medqa_4options_orig_filtered", "medqa_4options_g2b",
"MedQA: Orig vs G2B", "medqa_4options_orig_filtered", "medqa_4options_g2b"),
elem_id="scatter2"
)
with gr.Row():
scatter3 = gr.Plot(
value=create_scatter_plot(df, "b4bqa", "b4b",
"b4bqa vs b4b", "b4bqa", "b4b"),
elem_id="scatter3"
)
with gr.TabItem("📝 About"):
gr.Markdown(
"""<div style="text-align: center;">
<h2>About RABBITS LLM Leaderboard</h2>
<p>This leaderboard ...</p>
<p>It is designed to ...</p>
</div>""",
elem_classes="markdown-text"
)
with gr.TabItem("🚀 Submit Here!"):
gr.Markdown(
"""<div style="text-align: center;">
<h2>Submit Your Model Results</h2>
<p>If you have new model results that you would like to add to the leaderboard, please follow the submission guidelines below:</p>
<ul>
<li>COMING SOON</li>
</ul>
<p>COMING SOON</p>
</div>""",
elem_classes="markdown-text"
)
if __name__ == "__main__":
demo.launch()
|