"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space."""
import ast
import argparse
import glob
import pickle
import plotly
import gradio as gr
import numpy as np
import pandas as pd
import gradio as gr
import pandas as pd
import difflib
from pathlib import Path
from difflib import Differ
import json
from constants import BANNER, CITATION_TEXT, WINRATE_HEATMAP, css, js_code, all_task_types, DEFAULT_LP, TASK_TYPE_STR, js_light
from datetime import datetime, timezone
from data_utils import load_eval_results, sample_an_eval_result, apply_length_penalty, post_processing, add_winrates, add_winrates_tasks
# from gradio.themes.utils import colors, fonts, sizes
from themes import Seafoam
import datasets
from huggingface_hub import HfApi
# from datasets import Dataset, load_dataset, concatenate_datasets
import os, uuid
from utils_display import model_info
from tqdm import tqdm
from collections import defaultdict
from vapo_utils import get_diff_labels_for_demo, diff_texts
# get the last updated time from the elo_ranks.all.jsonl file
LAST_UPDATED = None
with open("_intro.md", "r") as f:
INTRO_MD = f.read()
with open("_about_us.md", "r") as f:
ABOUT_MD = f.read()
with open("_header.md", "r") as f:
HEADER_MD = f.read()
original_df, ablation_df = None, None
eval_results = load_eval_results()
available_models = [] # to be filled in later
import random
random.seed(42)
np.random.seed(42)
def sample_an_feedback(search_id, search_key_words, task_category, task_difficulty, task_quality, feedback_score, revision_better):
# print(f"search_id: {search_id}")
# print(f"search_key_words: {search_key_words}")
# print(f"task_category: {task_category}")
# print(f"task_difficulty: {task_difficulty}")
# print(f"task_quality: {task_quality}")
# print(f"feedback_score: {feedback_score}")
# print(f"revision_better: {revision_better}")
filter_task_category = set(task_category) < set(available_categories)
filter_task_difficulty = set(task_difficulty) < set(avaliable_difficulty)
filter_task_quality = set(task_quality) < set(avaliable_quality)
filter_feedback_score = set(feedback_score) < set(available_feedback_scores)
filter_revision_better = set(revision_better) < set(available_revision_better)
# print(f"filter_task_category: {filter_task_category}")
# print(f"filter_task_difficulty: {filter_task_difficulty}")
# print(f"filter_task_quality: {filter_task_quality}")
# print(f"filter_feedback_score: {filter_feedback_score}")
# print(f"filter_revision_better: {filter_revision_better}")
def filter_examples(item):
if not task_category or (filter_task_category and item['category'] not in task_category):
return False
if not task_difficulty or (filter_task_difficulty and item['difficulty'] not in task_difficulty):
return False
if not task_quality or (filter_task_quality and item['quality'] not in task_quality):
return False
if not feedback_score or (filter_feedback_score and item['feedback']['processed']['score'] not in feedback_score):
return False
if not revision_better or (filter_revision_better and item['pair_feedback']['revision_better'] not in revision_better):
return False
if search_id and item['id'] != search_id:
return False
if search_key_words:
if not all([key_word in item['query'] for key_word in search_key_words]):
return False
return True
valid_examples = dataset.filter(filter_examples, num_proc=4)
dummy_result = {
"session_id": "N/A",
"category": "N/A",
"difficulty": "N/A",
"quality": "N/A",
"intent": "N/A",
"ori_feedback": defaultdict(lambda: "N/A"),
"revision_better": "N/A",
"plan_history": {"user": ["N/A"], "assistant": ["N/A"]},
"ground_history": {"user": ["N/A"], "assistant": ["N/A"]},
"num_matches": 0,
"pred": "N/A",
"answer": "N/A",
"correctness": "N/A",
"pair_feedback_model": "N/A",
"image": "N/A"
}
if len(valid_examples) == 0:
gr.Warning("No examples found for the selected filters. Please try different filters.")
return dummy_result
print(f"Found {len(valid_examples)} examples for the selected filters.")
example = random.choice(valid_examples)
plan_history = {
"user": [
example['query'],
],
"assistant": [
example['response']
]
}
ground_history = {
"user": [
example['query'],
],
"assistant": [
example['revision']['processed']
]
}
result_dict = {
"session_id": example['id'],
"category": example['category'],
"difficulty": example['difficulty'],
"quality": example['quality'],
"intent": example['intent'],
"ori_feedback": example['feedback']['processed'],
"revision_better": example['pair_feedback']['revision_better'],
"plan_history": plan_history,
"ground_history": ground_history,
"num_matches": len(valid_examples),
# "pred": str(model_response_1['feedback']['processed']['score']) if model_response_1['feedback']['processed'] else "A",
# "answer": str(model_response_2['feedback']['processed']['score']) if model_response_2['feedback']['processed'] else "A",
"pred": example['model'], # model that generates the original response
"answer": example['revision']['model'], # model that generates the revised response
"correctness": example['feedback']['model'], # model that generates the feedback for the original response
"pair_feedback_model": example['pair_feedback']['model'], # model that generates the feedback for the revised response
"image": "file/data_dir/test_images/000000341196.jpg"
}
return result_dict
def display_chat_history(search_id, search_key_words, task_category, task_difficulty, task_quality, feedback_score, revision_better):
eval_item = sample_an_feedback(search_id, search_key_words, task_category, task_difficulty, task_quality, feedback_score, revision_better)
# eval_item = sample_an_feedback()
session_id = eval_item["session_id"]
category = eval_item["category"]
prediction = eval_item["pred"]
gold_answer = eval_item["answer"]
correctness = eval_item["correctness"]
difficulty = eval_item["difficulty"]
quality = eval_item["quality"]
intent = eval_item["intent"]
feedback = eval_item["ori_feedback"]
pair_feedback_model = eval_item["pair_feedback_model"]
if eval_item["image"]:
image_path = eval_item["image"]
else:
image_path = ""
chats_plan = []
for item_user, item_asst in zip(eval_item["plan_history"]["user"], eval_item["plan_history"]["assistant"]):
chats_plan += [item_user, item_asst]
chats_ground = []
for item_user, item_asst in zip(eval_item["ground_history"]["user"], eval_item["ground_history"]["assistant"]):
chats_ground += [item_user, item_asst]
chats_plan = [(chats_plan[i], chats_plan[i+1]) for i in range(0, len(chats_plan), 2)]
chats_ground = [(chats_ground[i], chats_ground[i+1]) for i in range(0, len(chats_ground), 2)]
match_ratio = f"{eval_item['num_matches']}/{len(dataset)} ({round(eval_item['num_matches'] / len(dataset) * 100, 2)} %)"
task_metadata = f"- 🆔: `{session_id}` \n- **Category**: {category} \n- **Difficulty**: {difficulty} \n- **Quality**: {quality} \n- **Intent**: {intent} \n- **Revision Better**: {eval_item['revision_better']} \n- **\#Matched items / \#Total Items**: {match_ratio}"
diff_text = diff_texts(chats_plan[-1][1], chats_ground[-1][1])
ori_labels, rev_labels = get_diff_labels_for_demo(chats_plan[-1][1], chats_ground[-1][1])
# print(f"Category: {category}")
# print(f"Difficulty: {difficulty}")
# print(f"Quality: {quality}")
# print(f"Intent: {intent}")
# print(f"Session ID: {session_id}")
# print(f"Original Response: {chats_plan}")
# print(f"Revised Response: {chats_ground}")
if image_path != "":
image = f''
return category, chats_plan, chats_ground, task_metadata, prediction, gold_answer, correctness, pair_feedback_model, image, diff_text, feedback['intent'], feedback['checklist'], feedback['strengths'], feedback['weaknesses'], feedback['score'], ori_labels, rev_labels
else:
return category, chats_plan, chats_ground, task_metadata, prediction, gold_answer, correctness, pair_feedback_model, f'
', diff_text, feedback['intent'], feedback['checklist'], feedback['strengths'], feedback['weaknesses'], feedback['score'], ori_labels, rev_labels
def slider_change_main(length_penalty):
global original_df, ablation_df
adjusted_df = apply_length_penalty(original_df, ablation_df, length_penalty)
adjusted_df = adjusted_df[["Model", "Overall Elo", "Task-Avg Elo", "# battles", "Length"]]
adjusted_df = adjusted_df.sort_values(by="Overall Elo", ascending=False)
adjusted_df = add_winrates(adjusted_df)
adjusted_df = adjusted_df.drop(columns=["Length"])
return adjusted_df
def slider_change_full(length_penalty, show_winrate):
global original_df, ablation_df
adjusted_df = apply_length_penalty(original_df, ablation_df, length_penalty)
# sort the model by the "Task-Avg Elo" column
adjusted_df = adjusted_df.sort_values(by="Task-Avg Elo", ascending=False)
adjusted_df.drop(columns=["Overall Elo", "Task-Avg Elo", "# battles", "Length"], inplace=True)
if show_winrate == "none":
return adjusted_df
elif show_winrate == "gpt-3.5":
adjusted_df = add_winrates_tasks(adjusted_df, ref="gpt-3.5")
elif show_winrate == "gpt-4":
adjusted_df = add_winrates_tasks(adjusted_df, ref="gpt-4")
return adjusted_df
seafoam = Seafoam()
def build_demo(TYPES):
global available_categories, avaliable_difficulty, avaliable_quality, available_feedback_scores, available_revision_better
with gr.Blocks(theme=gr.themes.Soft(), css=css, js=js_light) as demo:
gr.Markdown(HEADER_MD, elem_classes="markdown-text")
with gr.Tabs(elem_classes="tab-buttons") as tabs:
with gr.TabItem("🔍 Explore", elem_id="od-benchmark-tab-table", id=2):
with gr.Row():
btn_show_history = gr.Button("🎲 Click here to sample an example of Feedbacks ", elem_classes="sample_button")
with gr.Row():
with gr.Column():
with gr.Accordion("Search through ID", open=False, elem_classes="accordion-label"):
search_id = gr.Textbox("", label="Session ID", lines=1, max_lines=1, elem_classes="markdown-text-tiny")
search_id_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
search_id_clear_button.click(lambda: {search_id: {"value": "", "__type__": "update"}}, inputs=[], outputs=[search_id])
dummy_category = gr.Textbox(visible=False, label="Category")
dummy_difficulty = gr.Textbox(visible=False, label="Difficulty")
dummy_query = gr.Textbox(visible=False, label="Query")
dummy_feedback = gr.Textbox(visible=False, label="Feedback Score")
dummy_revision_better = gr.Textbox(visible=False, label="Revision Better")
gr.Examples(
[[x['id'], x['category'], x['difficulty'], x['query'], x['feedback']['processed']['score'], x['pair_feedback']['revision_better']] for x in highlighted_examples],
inputs=[search_id, dummy_category, dummy_difficulty, dummy_query, dummy_feedback, dummy_revision_better],
label="Highlighted Examples",
examples_per_page=5,
)
with gr.Accordion("Search through multiple keywords in the query", open=False, elem_classes="accordion-label"):
search_key_words = gr.Dropdown(allow_custom_value=True, multiselect=True, label="Keywords (press enter to confirm)", elem_classes="markdown-text-tiny")
search_key_words_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
search_key_words_clear_button.click(lambda: {search_key_words: {"value": [], "__type__": "update"}}, inputs=[], outputs=[search_key_words])
with gr.Accordion("Choose task difficulty", open=False, elem_classes="accordion-label"):
selected_task_difficulty = gr.CheckboxGroup(avaliable_difficulty, info="", value=avaliable_difficulty, show_label=False, elem_id="select-difficulty")
selected_task_difficulty_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
selected_task_difficulty_clear_button.click(lambda: {selected_task_difficulty: {"value": [], "__type__": "update"}}, inputs=[], outputs=[selected_task_difficulty])
with gr.Accordion("Choose task quality", open=False, elem_classes="accordion-label"):
selected_task_quality = gr.CheckboxGroup(avaliable_quality, info="", value=avaliable_quality, show_label=False, elem_id="select-quality")
selected_task_quality_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
selected_task_quality_clear_button.click(lambda: {selected_task_quality: {"value": [], "__type__": "update"}}, inputs=[], outputs=[selected_task_quality])
with gr.Accordion("Choose task category", open=False, elem_classes="accordion-label"):
selected_task_category = gr.CheckboxGroup(available_categories, info="", value=available_categories, show_label=False, elem_id="select-category")
selected_task_category_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
selected_task_category_clear_button.click(lambda: {selected_task_category: {"value": [], "__type__": "update"}}, inputs=[], outputs=[selected_task_category])
with gr.Accordion("Choose feedback score for original response", open=False, elem_classes="accordion-label"):
selected_feedback_score = gr.CheckboxGroup(available_feedback_scores, info="", value=available_feedback_scores, show_label=False, elem_id="select-feedback")
selected_feedback_score_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
selected_feedback_score_clear_button.click(lambda: {selected_feedback_score: {"value": [], "__type__": "update"}}, inputs=[], outputs=[selected_feedback_score])
with gr.Accordion("Choose whether the revised response is better than the original response", open=False, elem_classes="accordion-label"):
selected_revision_better = gr.CheckboxGroup(available_revision_better, info="", value=available_revision_better, show_label=False, elem_id="select-revision-better")
selected_revision_better_clear_button = gr.Button("Clear", elem_classes="btn_boderline_gray", scale=1)
selected_revision_better_clear_button.click(lambda: {selected_revision_better: {"value": [], "__type__": "update"}}, inputs=[], outputs=[selected_revision_better])
with gr.Row(visible=False):
with gr.Column(scale=1.5):
with gr.Accordion("📝 Task Description", open=True, elem_classes="accordion-label"):
task = gr.Markdown("", elem_classes="markdown-text-tiny")
task.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Column(scale=1):
with gr.Accordion("Input Image (optional)", open=True, elem_classes="accordion-label"):
image = gr.HTML("", elem_id="markdown-text-tiny")
image.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Row():
with gr.Column():
# with gr.Accordion("📝 Task Metadata", open=True, elem_classes="accordion-label"):
with gr.Accordion("📝 Item Metadata", open=True, elem_classes="accordion-label"):
task_metadata = gr.Markdown("", elem_classes="markdown-text-tiny")
task_metadata.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Row():
with gr.Column():
# with gr.Accordion("🙋 Prediction", open=True, elem_classes="accordion-label"):
with gr.Accordion("Policy Model", open=True, elem_classes="accordion-label"):
prediction = gr.HTML("", elem_classes="markdown-text-tiny")
prediction.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Column():
# with gr.Accordion("🔑 Ground-Truth Answer", open=True, elem_classes="accordion-label"):
with gr.Accordion("Revision Model", open=True, elem_classes="accordion-label"):
gold_answer = gr.HTML("", elem_id="markdown-text-tiny")
gold_answer.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Column(visible=True):
with gr.Accordion("Feedback Model", open=True, elem_classes="accordion-label"):
correctness = gr.HTML("", elem_id="markdown-text-tiny")
correctness.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Column(visible=True):
with gr.Accordion("Feedback Model (2nd stage)", open=True, elem_classes="accordion-label"):
pair_feedback_model = gr.HTML("", elem_id="markdown-text-tiny")
pair_feedback_model.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Row():
with gr.Column(scale=1):
# gr.Markdown("## 📢 Plan Module Process History w/ Execution Module Results", elem_classes="accordion-label")
gr.Markdown("## 📢 Policy Model Response (Original)", elem_classes="accordion-label")
Chatbot_Common_Plan = gr.Chatbot(avatar_images=["human_icon.jpeg", "ai_icon.png"], height=3000, container=False, label="Original Model Response", likeable=False, show_share_button=False, show_label=True, elem_classes="chat-common", layout="bubble")
Chatbot_Common_Plan.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Column(scale=1):
# gr.Markdown("## 📢 Ground Module Process History", elem_classes="accordion-label")
gr.Markdown("## 📢 Revision Model Response (Revised)", elem_classes="accordion-label")
Chatbot_Common_Ground = gr.Chatbot(avatar_images=["human_icon.jpeg", "ai_icon.png"], height=3000, container=False, label="Revised Model Response", likeable=False, show_share_button=False, show_label=True, elem_classes="chat-common", layout="bubble")
Chatbot_Common_Ground.change(lambda x: x, inputs=[], outputs=[], scroll_to_output=False, js=js_code)
with gr.Row():
with gr.Column():
with gr.Accordion("📊 Feedback of the original response", open=True, elem_classes="accordion-label"):
intent = gr.Textbox("", lines=1, max_lines=30, label="Intent", elem_classes="markdown-text-tiny")
checklist = gr.Textbox("", lines=1, max_lines=30, label="Checklist", elem_classes="markdown-text-tiny")
strengths = gr.Textbox("", lines=1, max_lines=30, label="Strengths", elem_classes="markdown-text-tiny")
weaknesses = gr.Textbox("", lines=1, max_lines=30, label="Weaknesses", elem_classes="markdown-text-tiny")
feedback_score = gr.Textbox("", lines=1, max_lines=1, label="Feedback Score", elem_classes="markdown-text-tiny")
with gr.Column():
with gr.Accordion("Highlighted differences", open=True, elem_classes="accordion-label"):
highlighted_diff = gr.HighlightedText(label="Original (-) vs Revised (+)",
combine_adjacent=False,
show_legend=True,
color_map={"-": "red", "+": "green"})
with gr.Row():
with gr.Column():
with gr.Accordion("Labels of original response (rejected response) ", open=True, elem_classes="accordion-label"):
ori_labels = gr.HighlightedText(label="Labels (-)",
combine_adjacent=False,
show_legend=True,
color_map={"-": "red", "+": "green", "#": "blue"})
with gr.Column():
with gr.Accordion("Labels of revised response (accepted response)", open=True, elem_classes="accordion-label"):
rev_labels = gr.HighlightedText(label="Labels (+)",
combine_adjacent=False,
show_legend=True,
color_map={"-": "red", "+": "green", "#": "blue"})
# Display chat history when button is clicked
btn_show_history.click(fn=display_chat_history,
inputs=[search_id, search_key_words, selected_task_category, selected_task_difficulty, selected_task_quality, selected_feedback_score, selected_revision_better],
outputs=[
task, Chatbot_Common_Plan, Chatbot_Common_Ground, task_metadata,
prediction, gold_answer, correctness, pair_feedback_model,
image, highlighted_diff,
intent, checklist, strengths, weaknesses, feedback_score,
ori_labels, rev_labels
])
with gr.TabItem("📊 Templates", elem_id="od-benchmark-tab-table", id=3):
leading_text = """Here are the templates we used in the VAPO framework.
To comment on the template, click the following links:
- [Feedback template (1-shot)](https://hackmd.io/@nVvBa9WFT4SUUDHwIWaP8w/Sy7YEKuKC)
- [Revision template (0-shot)](https://hackmd.io/@nVvBa9WFT4SUUDHwIWaP8w/rkhgLKOFC)
- [2nd Stage Feedback template (Pairwise)](https://hackmd.io/@nVvBa9WFT4SUUDHwIWaP8w/By8QUtOKC)"""
gr.Markdown(leading_text, elem_classes="markdown-text")
with gr.TabItem("📮 About Us", elem_id="od-benchmark-tab-table", id=4, visible=False):
gr.Markdown(ABOUT_MD, elem_classes="markdown-text")
gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text-small")
with gr.Row():
with gr.Accordion("📙 Citation", open=False, elem_classes="accordion-label", visible=False):
gr.Textbox(
value=CITATION_TEXT,
lines=7,
label="Copy the BibTeX snippet to cite this source",
elem_id="citation-button",
show_copy_button=True)
# ).style(show_copy_button=True)
return demo
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--share", action="store_true")
parser.add_argument("--result_file", help="Path to results table", default="data_dir/pair_feedbacks_2.jsonl")
parser.add_argument("--length_balation_file", help="Path to results table", default="data_dir/elo_ranks.length_ablation.all.jsonl")
parser.add_argument("--skip_empty_result_file", help="Path to results table", default="data_dir/elo_ranks.skip_empty.all.jsonl")
parser.add_argument("--skip_empty_length_balation_file", help="Path to results table", default="data_dir/elo_ranks.skip_empty.length_ablation.all.jsonl")
args = parser.parse_args()
# LAST_UPDATED = datetime.fromtimestamp(Path(args.result_file).stat().st_mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
# available_models = sorted(list(set(list(original_df["model name "]))))
# available_models = list(model_info.keys())
# dataset = datasets.Dataset.from_json(args.result_file)
dataset = datasets.load_dataset("DongfuJiang/VAPO", "pair_feedback_iter_1", split='train')
avaliable_difficulty = sorted(list(set(dataset['difficulty'])))
avaliable_quality = sorted(list(set(dataset['quality'])))
available_feedback_scores = sorted(list(set([item['feedback']['processed']['score'] for item in dataset])))
available_categories = sorted(list(set(dataset['category'])))
available_revision_better = sorted(list(set([item['pair_feedback']['revision_better'] for item in dataset])))
with open('./highlighted_ids.txt', 'r') as f:
highlighted_ids = f.read().splitlines()
highlighted_examples = dataset.filter(lambda x: x['id'] in highlighted_ids, num_proc=4)
TYPES = ["markdown", "number"]
demo = build_demo(TYPES)
# demo.launch(share=args.share, allowed_paths=["."], height=1000, server_port=13133)
demo.launch(share=args.share, allowed_paths=["."], height=1000)