import os import sys sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./leaderboard/evaluation"))) sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./leaderboard"))) os.chdir(os.path.dirname(os.path.abspath(__file__))) os.environ['CURL_CA_BUNDLE'] = '' import json import datetime from email.utils import parseaddr import gradio as gr import pandas as pd import numpy as np from datasets import load_dataset from apscheduler.schedulers.background import BackgroundScheduler from huggingface_hub import HfApi # InfoStrings # from scorer import question_scorer from content import format_error, format_warning, format_log, TITLE, INTRODUCTION_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, model_hyperlink from eval import eval_score TOKEN = os.environ.get("TOKEN", None) OWNER="osunlp" DATA_DATASET = f"{OWNER}/TravelPlanner" EVAL_DATASET = f"{OWNER}/TravelPlannerEval" RESULTS_DATASET = f"{OWNER}/TravelPlannerPublicResults" api = HfApi() # 'scores' = "2024" os.makedirs("scored", exist_ok=True) # # Display the results eval_results = load_dataset(RESULTS_DATASET, 'scores', token=TOKEN, download_mode="force_redownload", ignore_verifications=True) def get_dataframe_from_results(eval_results, split, mode): local_df = eval_results[f'{split}_{mode}'] local_df = local_df.remove_columns(["Mail"]) df = pd.DataFrame(local_df) df['Organization'].mask(df['Organization']=='TravelBench Team','TravelPlanner Team',inplace=True) df = df.sort_values(by=["Final Pass Rate"], ascending=False) numeric_cols = [c for c in local_df.column_names if "Rate" in c] df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2) return df eval_dataframe_val_twostage = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='twostage') eval_dataframe_val_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='soleplanning') eval_dataframe_test_twostage = get_dataframe_from_results(eval_results=eval_results, split="test",mode='twostage') eval_dataframe_test_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="test",mode='soleplanning') # def restart_space(): # api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN) def load_line_json_data(filename): data = [] with open(filename, 'r', encoding='utf-8') as f: for line in f.read().strip().split('\n'): unit = json.loads(line) data.append(unit) return data def add_new_eval( val_or_test: str, eval_mode: str, model: str, tooluse_strategy: str, planning_strategy: str, organization: str, mail: str, path_to_file: str, ): # Very basic email parsing _, parsed_mail = parseaddr(mail) if not "@" in parsed_mail: return format_warning("Please provide a valid email adress.") print("Adding new eval") if path_to_file is None: return format_warning("Please attach a file.") # Save submitted file api.upload_file( repo_id=RESULTS_DATASET, path_or_fileobj=path_to_file.name, path_in_repo=f"{organization}/{val_or_test}_{eval_mode}_{tooluse_strategy}_{planning_strategy}_raw_{datetime.datetime.today()}.jsonl", repo_type="dataset", token=TOKEN ) # Compute score file_path = path_to_file.name result = eval_score(val_or_test,file_path=file_path,TOKEN=TOKEN) with open(f"scored/{organization}_{val_or_test}_{eval_mode}_{tooluse_strategy}_{planning_strategy}.jsonl", "w") as scored_file: scored_file.write(json.dumps(result) + "\n") # Save scored file api.upload_file( repo_id=RESULTS_DATASET, path_or_fileobj=f"scored/{organization}_{val_or_test}_{eval_mode}_{tooluse_strategy}_{planning_strategy}.jsonl", path_in_repo=f"{organization}/{model}/{val_or_test}_{eval_mode}_{tooluse_strategy}_{planning_strategy}_scored_{datetime.datetime.today()}.jsonl", repo_type="dataset", token=TOKEN ) # Actual submission eval_entry = { "Model": model, "Tool-use Strategy": tooluse_strategy, "Planning Strategy": planning_strategy, "Organization": organization, "Mail": mail, "Delivery Rate": result['Delivery Rate'], "Commonsense Constraint Micro Pass Rate":result['Commonsense Constraint Micro Pass Rate'], "Commonsense Constraint Macro Pass Rate":result['Commonsense Constraint Macro Pass Rate'], "Hard Constraint Micro Pass Rate":result['Hard Constraint Micro Pass Rate'], "Hard Constraint Macro Pass Rate":result['Hard Constraint Macro Pass Rate'], "Final Pass Rate":result['Final Pass Rate'] } eval_mode = eval_mode.replace('-','') eval_results[f'{val_or_test}_{eval_mode}'] = eval_results[f'{val_or_test}_{eval_mode}'].add_item(eval_entry) print(eval_results) eval_results.push_to_hub(RESULTS_DATASET, config_name = 'scores', token=TOKEN) return format_log(f"Model: {model} | Tool-use Strategy: {tooluse_strategy} | Planning Strategy: {planning_strategy} | submitted by {organization} successfully. \nPlease refresh the leaderboard, and wait a bit to see the score displayed (Validation ~2mins, Test ~7mins).") def refresh(): eval_results = load_dataset(RESULTS_DATASET, 'scores', token=TOKEN, download_mode="force_redownload", ignore_verifications=True) eval_dataframe_val_twostage = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='twostage') eval_dataframe_val_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='soleplanning') eval_dataframe_test_twostage = get_dataframe_from_results(eval_results=eval_results, split="test",mode='twostage') eval_dataframe_test_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="test",mode='soleplanning') return eval_dataframe_val_twostage, eval_dataframe_val_soleplanning, eval_dataframe_test_twostage, eval_dataframe_test_soleplanning # def upload_file(files): # file_paths = [file.name for file in files] # return file_paths demo = gr.Blocks() with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tab("Results: Validation | Two-Stage "): leaderboard_table_val_twostage = gr.components.Dataframe( value=eval_dataframe_val_twostage, interactive=False, ) with gr.Tab("Results: Validation | Sole-Planning"): leaderboard_table_val_soleplanning = gr.components.Dataframe( value=eval_dataframe_val_soleplanning, interactive=False, ) with gr.Tab("Results: Test | Two-Stage "): leaderboard_table_test_twostage = gr.components.Dataframe( value=eval_dataframe_test_twostage, interactive=False, ) with gr.Tab("Results: Test | Sole-Planning"): leaderboard_table_test_soleplanning = gr.components.Dataframe( value=eval_dataframe_test_soleplanning, interactive=False, ) refresh_button = gr.Button("Refresh") refresh_button.click( refresh, inputs=[], outputs=[ leaderboard_table_val_twostage, leaderboard_table_val_soleplanning, leaderboard_table_test_twostage, leaderboard_table_test_soleplanning, ], ) with gr.Accordion("Submit a new file for evaluation"): with gr.Row(): with gr.Column(): level_of_test = gr.Radio(["validation", "test"], value="validation", label="Split") eval_mode = gr.Radio(["two-stage", "sole-planning"], value="two-stage", label="Eval Mode") model = gr.Textbox(label="Foundation Model") tooluse_strategy = gr.Textbox(label="Tool-use Strategy") planning_strategy = gr.Textbox(label="Planning Strategy") with gr.Column(): organization = gr.Textbox(label="Organization") mail = gr.Textbox(label="Contact email") file_output = gr.File() submit_button = gr.Button("Submit Eval") submission_result = gr.Markdown() submit_button.click( add_new_eval, [ level_of_test, eval_mode, model, tooluse_strategy, planning_strategy, organization, mail, file_output, ], submission_result, ) demo.launch(debug=True)