Sampler-Arena / app.py
rwitz's picture
Update app.py
4db4ec7
raw
history blame
8.66 kB
import gradio as gr
import requests
import os
#
import json
import random
from elo import update_elo_ratings # Custom function for ELO ratings
enable_btn = gr.Button.update(interactive=True)
# Load chatbot URLs and model names from a JSON file
with open('chatbot_urls.json', 'r') as file:
chatbots = json.load(file)
def clear_chat(state):
# Reset state including the chatbot order
state = {} if state is not None else state
# Shuffle and reinitialize chatbots in the state
bot_names = list(chatbots.keys())
random.shuffle(bot_names)
state['last_bots'] = [bot_names[0], bot_names[1]]
# Reset other components
return state, None, None, gr.Button.update(interactive=False), gr.Button.update(interactive=False), gr.Textbox.update(interactive=False), gr.Button.update(interactive=False)
global_elo_ratings=None
from datasets import load_dataset,DatasetDict
import requests
import os
def get_user_elo_ratings():
global global_elo_ratings
dataset = load_dataset("rwitz/mistral-elo-ratings", download_mode="force_redownload")
global_elo_ratings = dataset['train'] # or the relevant split
return global_elo_ratings
def update_elo_rating(new_rating,winner,loser):
# Fetch the current dataset
elo_ratings = get_user_elo_ratings()
# Function to update the rating of a specific player
# Update the dataset
# Convert updated dataset to a dictionary for pushing
updated_ratings=update_elo_ratings(elo_ratings,winner,loser)
updated_ratings.push_to_hub("rwitz/mistral-elo-ratings",token=os.environ.get("huggingface_token"))
# Function to get bot response
def format_alpaca_prompt(state):
alpaca_prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request."
alpaca_prompt2 = "Below is an instruction that describes a task. Write a response that appropriately completes the request."
for message in state["history"][0]:
j=""
if message['role']=='user':
j="### Instruction:\n"
else:
j="### Response:\n"
alpaca_prompt += j+ message['content']+"\n\n"
for message in state["history"][1]:
j=""
if message['role']=='user':
j="### Instruction:\n"
else:
j="### Response:\n"
alpaca_prompt2 += j+ message['content']+"\n\n"
return [alpaca_prompt+"### Response:\n",alpaca_prompt2+"### Response:\n"]
def get_bot_response(url, prompt,state,bot_index):
alpaca_prompt = format_alpaca_prompt(state)
payload = {
"input": {
"prompt": alpaca_prompt[bot_index],
"sampling_params": {
"max_new_tokens": 50,
"temperature": 0.7,
"top_p":0.95
}
}
}
headers = {
"accept": "application/json",
"content-type": "application/json",
"authorization": os.environ.get("RUNPOD_TOKEN")
}
response = requests.post(url, json=payload, headers=headers)
return response.json()['output'].split('### Instruction')[0]
def chat_with_bots(user_input, state):
# Use existing bot order from state if available, otherwise shuffle and initialize
if 'last_bots' not in state or not state['last_bots']:
bot_names = list(chatbots.keys())
random.shuffle(bot_names)
state['last_bots'] = [bot_names[0], bot_names[1]]
bot1_url, bot2_url = chatbots[state['last_bots'][0]], chatbots[state['last_bots'][1]]
bot1_response = get_bot_response(bot1_url, user_input, state, 0)
bot2_response = get_bot_response(bot2_url, user_input, state, 1)
return bot1_response, bot2_response
def update_ratings(state, winner_index):
elo_ratings = get_user_elo_ratings()
bot_names = list(chatbots.keys())
winner = state['last_bots'][winner_index]
loser = state['last_bots'][1 - winner_index]
elo_ratings = update_elo_ratings(elo_ratings, winner, loser)
update_elo_rating(elo_ratings,winner,loser)
return [('Winner: ',state['last_bots'][winner_index]),('Loser: ',state['last_bots'][1 - winner_index])]
def vote_up_model(state, chatbot,chatbot2):
update_message = update_ratings(state, 0)
chatbot.append(update_message[0])
chatbot2.append(update_message[1])
return chatbot, chatbot2,gr.Button.update(interactive=False),gr.Button.update(interactive=False),gr.Textbox.update(interactive=False),gr.Button.update(interactive=False) # Disable voting buttons
def vote_down_model(state, chatbot,chatbot2):
update_message = update_ratings(state, 1)
chatbot2.append(update_message[0])
chatbot.append(update_message[1])
return chatbot,chatbot2, gr.Button.update(interactive=False),gr.Button.update(interactive=False),gr.Textbox.update(interactive=False),gr.Button.update(interactive=False) # Disable voting buttons
def user_ask(state, chatbot1, chatbot2, textbox):
global enable_btn
user_input = textbox
if len(user_input) > 200:
user_input = user_input[:200] # Limit user input to 200 characters
# Updating state with the current ELO ratings
state["elo_ratings"] = get_user_elo_ratings()
if "history" not in state:
state.update({'history': [[],[]]})
state["history"][0].extend([
{"role": "user", "content": user_input}])
state["history"][1].extend([
{"role": "user", "content": user_input}])
# Chat with bots
bot1_response, bot2_response = chat_with_bots(user_input, state)
state["history"][0].extend([
{"role": "bot1", "content": bot1_response},
])
state["history"][1].extend([
{"role": "bot2", "content": bot2_response},
])
chatbot1.append((user_input,bot1_response))
chatbot2.append((user_input,bot2_response))
# Keep only the last 10 messages in history
state["history"] = state["history"][-10:]
# Format the conversation in ChatML format
return state, chatbot1, chatbot2, gr.update(value=''),enable_btn,enable_btn
import pandas as pd
# Function to generate leaderboard data
def generate_leaderboard():
global global_elo_ratings
global_elo_ratings = get_user_elo_ratings()
# Convert the Hugging Face dataset to a pandas DataFrame
leaderboard_data = pd.DataFrame(global_elo_ratings)
# Rename columns to 'Chatbot' and 'ELO Score'
leaderboard_data.columns = ['Chatbot', 'ELO Score']
# Round the ELO Score to the nearest whole number
leaderboard_data['ELO Score'] = leaderboard_data['ELO Score'].round()
# Sort the DataFrame based on the ELO Score in descending order
leaderboard_data = leaderboard_data.sort_values('ELO Score', ascending=False)
return leaderboard_data
def refresh_leaderboard():
return generate_leaderboard()
# Gradio interface setup
with gr.Blocks() as demo:
state = gr.State({})
with gr.Tab("Chatbot Arena"):
with gr.Row():
with gr.Column():
chatbot1 = gr.Chatbot(label='Model A').style(height=600)
upvote_btn_a = gr.Button(value="πŸ‘ Upvote A",interactive=False)
with gr.Column():
chatbot2 = gr.Chatbot(label='Model B').style(height=600)
upvote_btn_b = gr.Button(value="πŸ‘ Upvote B",interactive=False)
textbox = gr.Textbox(placeholder="Enter your prompt (up to 200 characters)", max_chars=200)
with gr.Row():
submit_btn = gr.Button(value="Send")
reset_btn = gr.Button(value="Reset")
reset_btn.click(clear_chat, inputs=[state], outputs=[state, chatbot1, chatbot2, upvote_btn_a, upvote_btn_b,textbox,submit_btn])
textbox.submit(user_ask, inputs=[state, chatbot1, chatbot2, textbox], outputs=[state, chatbot1, chatbot2, textbox,upvote_btn_a,upvote_btn_b])
submit_btn.click(user_ask, inputs=[state, chatbot1, chatbot2, textbox], outputs=[state, chatbot1, chatbot2, textbox,upvote_btn_a,upvote_btn_b])
upvote_btn_a.click(vote_up_model, inputs=[state, chatbot1,chatbot2], outputs=[chatbot1,chatbot2,upvote_btn_a,upvote_btn_b,textbox,submit_btn])
upvote_btn_b.click(vote_down_model, inputs=[state, chatbot1,chatbot2], outputs=[chatbot1,chatbot2,upvote_btn_a,upvote_btn_b,textbox,submit_btn])
with gr.Tab("Leaderboard"):
leaderboard = gr.Dataframe(refresh_leaderboard())
refresh_btn = gr.Button("Refresh Leaderboard")
# Function to refresh leaderboard
# Event handler for the refresh button
refresh_btn.click(refresh_leaderboard, inputs=[], outputs=[leaderboard])
# Launch the Gradio interface
if __name__ == "__main__":
demo.launch(share=False)